index/zrpn.c

   1 /* $Id: zrpn.c,v 1.180 2005-04-29 10:36:13 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #else
  28 #include <unistd.h>
  29 #endif
  30 #include <ctype.h>
  31
  32 #include <yaz/diagbib1.h>
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39 static const struct key_control it_ctrl =
  40 {
  41     sizeof(struct it_key),
  42     2, /* we have sysnos and seqnos in this key, nothing more */
  43     key_compare_it,
  44     key_logdump_txt,   /* FIXME  - clean up these functions */
  45     key_get_seq,
  46 };
  47
  48
  49 const struct key_control *key_it_ctrl = &it_ctrl;
  50
  51 struct rpn_char_map_info
  52 {
  53     ZebraMaps zm;
  54     int reg_type;
  55 };
  56
  57 typedef struct
  58 {
  59     int type;
  60     int major;
  61     int minor;
  62     Z_AttributesPlusTerm *zapt;
  63 } AttrType;
  64
  65
  66 static int log_level_set = 0;
  67 static int log_level_rpn = 0;
  68
  69 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  70 {
  71     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  72     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  73 #if 0
  74     if (out && *out)
  75     {
  76         const char *outp = *out;
  77         yaz_log(YLOG_LOG, "---");
  78         while (*outp)
  79         {
  80             yaz_log(YLOG_LOG, "%02X", *outp);
  81             outp++;
  82         }
  83     }
  84 #endif
  85     return out;
  86 }
  87
  88 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  89                                   struct rpn_char_map_info *map_info)
  90 {
  91     map_info->zm = reg->zebra_maps;
  92     map_info->reg_type = reg_type;
  93     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  94 }
  95
  96 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  97                          const char **string_value)
  98 {
  99     int num_attributes;
 100
 101     num_attributes = src->zapt->attributes->num_attributes;
 102     while (src->major < num_attributes)
 103     {
 104         Z_AttributeElement *element;
 105
 106         element = src->zapt->attributes->attributes[src->major];
 107         if (src->type == *element->attributeType)
 108         {
 109             switch (element->which)
 110             {
 111             case Z_AttributeValue_numeric:
 112                 ++(src->major);
 113                 if (element->attributeSet && attributeSetP)
 114                 {
 115                     oident *attrset;
 116
 117                     attrset = oid_getentbyoid(element->attributeSet);
 118                     *attributeSetP = attrset->value;
 119                 }
 120                 return *element->value.numeric;
 121                 break;
 122             case Z_AttributeValue_complex:
 123                 if (src->minor >= element->value.complex->num_list)
 124                     break;
 125                 if (element->attributeSet && attributeSetP)
 126                 {
 127                     oident *attrset;
 128
 129                     attrset = oid_getentbyoid(element->attributeSet);
 130                     *attributeSetP = attrset->value;
 131                 }
 132                 if (element->value.complex->list[src->minor]->which ==
 133                     Z_StringOrNumeric_numeric)
 134                 {
 135                     ++(src->minor);
 136                     return
 137                         *element->value.complex->list[src->minor-1]->u.numeric;
 138                 }
 139                 else if (element->value.complex->list[src->minor]->which ==
 140                          Z_StringOrNumeric_string)
 141                 {
 142                     if (!string_value)
 143                         break;
 144                     ++(src->minor);
 145                     *string_value =
 146                         element->value.complex->list[src->minor-1]->u.string;
 147                     return -2;
 148                 }
 149                 else
 150                     break;
 151             default:
 152                 assert(0);
 153             }
 154         }
 155         ++(src->major);
 156     }
 157     return -1;
 158 }
 159
 160 static int attr_find(AttrType *src, oid_value *attributeSetP)
 161 {
 162     return attr_find_ex(src, attributeSetP, 0);
 163 }
 164
 165 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 166                        int type)
 167 {
 168     src->zapt = zapt;
 169     src->type = type;
 170     src->major = 0;
 171     src->minor = 0;
 172 }
 173
 174 #define TERM_COUNT
 175
 176 struct grep_info {
 177 #ifdef TERM_COUNT
 178     int *term_no;
 179 #endif
 180     ISAM_P *isam_p_buf;
 181     int isam_p_size;
 182     int isam_p_indx;
 183     ZebraHandle zh;
 184     int reg_type;
 185     ZebraSet termset;
 186 };
 187
 188 static void term_untrans(ZebraHandle zh, int reg_type,
 189                            char *dst, const char *src)
 190 {
 191     int len = 0;
 192     while (*src)
 193     {
 194         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 195                                            reg_type, &src);
 196         if (!cp && len < IT_MAX_WORD-1)
 197             dst[len++] = *src++;
 198         else
 199             while (*cp && len < IT_MAX_WORD-1)
 200                 dst[len++] = *cp++;
 201     }
 202     dst[len] = '\0';
 203 }
 204
 205 static void add_isam_p(const char *name, const char *info,
 206                        struct grep_info *p)
 207 {
 208     if (!log_level_set)
 209     {
 210         log_level_rpn = yaz_log_module_level("rpn");
 211         log_level_set = 1;
 212     }
 213     if (p->isam_p_indx == p->isam_p_size)
 214     {
 215         ISAM_P *new_isam_p_buf;
 216 #ifdef TERM_COUNT
 217         int *new_term_no;
 218 #endif
 219         p->isam_p_size = 2*p->isam_p_size + 100;
 220         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 221                                             p->isam_p_size);
 222         if (p->isam_p_buf)
 223         {
 224             memcpy(new_isam_p_buf, p->isam_p_buf,
 225                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 226             xfree(p->isam_p_buf);
 227         }
 228         p->isam_p_buf = new_isam_p_buf;
 229
 230 #ifdef TERM_COUNT
 231         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 232         if (p->term_no)
 233         {
 234             memcpy(new_term_no, p->isam_p_buf,
 235                     p->isam_p_indx * sizeof(*p->term_no));
 236             xfree(p->term_no);
 237         }
 238         p->term_no = new_term_no;
 239 #endif
 240     }
 241     assert(*info == sizeof(*p->isam_p_buf));
 242     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 243
 244 #if 1
 245     if (p->termset)
 246     {
 247         const char *db;
 248         int set, use;
 249         char term_tmp[IT_MAX_WORD];
 250         int su_code = 0;
 251         int len = key_SU_decode (&su_code, name);
 252
 253         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 254         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 255         zebraExplain_lookup_ord (p->zh->reg->zei,
 256                                  su_code, &db, &set, &use);
 257         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 258
 259         resultSetAddTerm(p->zh, p->termset, name[len], db,
 260                          set, use, term_tmp);
 261     }
 262 #endif
 263     (p->isam_p_indx)++;
 264 }
 265
 266 static int grep_handle(char *name, const char *info, void *p)
 267 {
 268     add_isam_p(name, info, (struct grep_info *) p);
 269     return 0;
 270 }
 271
 272 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 273                     const char *ct1, const char *ct2, int first)
 274 {
 275     const char *s1, *s0 = *src;
 276     const char **map;
 277
 278     /* skip white space */
 279     while (*s0)
 280     {
 281         if (ct1 && strchr(ct1, *s0))
 282             break;
 283         if (ct2 && strchr(ct2, *s0))
 284             break;
 285         s1 = s0;
 286         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 287         if (**map != *CHR_SPACE)
 288             break;
 289         s0 = s1;
 290     }
 291     *src = s0;
 292     return *s0;
 293 }
 294
 295
 296 static void esc_str(char *out_buf, int out_size,
 297                     const char *in_buf, int in_size)
 298 {
 299     int k;
 300
 301     assert(out_buf);
 302     assert(in_buf);
 303     assert(out_size > 20);
 304     *out_buf = '\0';
 305     for (k = 0; k<in_size; k++)
 306     {
 307         int c = in_buf[k] & 0xff;
 308         int pc;
 309         if (c < 32 || c > 126)
 310             pc = '?';
 311         else
 312             pc = c;
 313         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 314         if (strlen(out_buf) > out_size-20)
 315         {
 316             strcat(out_buf, "..");
 317             break;
 318         }
 319     }
 320 }
 321
 322 #define REGEX_CHARS " []()|.*+?!"
 323
 324 /* term_100: handle term, where trunc = none(no operators at all) */
 325 static int term_100(ZebraMaps zebra_maps, int reg_type,
 326                     const char **src, char *dst, int space_split,
 327                     char *dst_term)
 328 {
 329     const char *s0;
 330     const char **map;
 331     int i = 0;
 332     int j = 0;
 333
 334     const char *space_start = 0;
 335     const char *space_end = 0;
 336
 337     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 338         return 0;
 339     s0 = *src;
 340     while (*s0)
 341     {
 342         const char *s1 = s0;
 343         int q_map_match = 0;
 344         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 345                                 &q_map_match);
 346         if (space_split)
 347         {
 348             if (**map == *CHR_SPACE)
 349                 break;
 350         }
 351         else  /* complete subfield only. */
 352         {
 353             if (**map == *CHR_SPACE)
 354             {   /* save space mapping for later  .. */
 355                 space_start = s1;
 356                 space_end = s0;
 357                 continue;
 358             }
 359             else if (space_start)
 360             {   /* reload last space */
 361                 while (space_start < space_end)
 362                 {
 363                     if (strchr(REGEX_CHARS, *space_start))
 364                         dst[i++] = '\\';
 365                     dst_term[j++] = *space_start;
 366                     dst[i++] = *space_start++;
 367                 }
 368                 /* and reset */
 369                 space_start = space_end = 0;
 370             }
 371         }
 372         /* add non-space char */
 373         memcpy(dst_term+j, s1, s0 - s1);
 374         j += (s0 - s1);
 375         if (!q_map_match)
 376         {
 377             while (s1 < s0)
 378             {
 379                 if (strchr(REGEX_CHARS, *s1))
 380                     dst[i++] = '\\';
 381                 dst[i++] = *s1++;
 382             }
 383         }
 384         else
 385         {
 386             char tmpbuf[80];
 387             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 388
 389             strcpy(dst + i, map[0]);
 390             i += strlen(map[0]);
 391         }
 392     }
 393     dst[i] = '\0';
 394     dst_term[j] = '\0';
 395     *src = s0;
 396     return i;
 397 }
 398
 399 /* term_101: handle term, where trunc = Process # */
 400 static int term_101(ZebraMaps zebra_maps, int reg_type,
 401                     const char **src, char *dst, int space_split,
 402                     char *dst_term)
 403 {
 404     const char *s0;
 405     const char **map;
 406     int i = 0;
 407     int j = 0;
 408
 409     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 410         return 0;
 411     s0 = *src;
 412     while (*s0)
 413     {
 414         if (*s0 == '#')
 415         {
 416             dst[i++] = '.';
 417             dst[i++] = '*';
 418             dst_term[j++] = *s0++;
 419         }
 420         else
 421         {
 422             const char *s1 = s0;
 423             int q_map_match = 0;
 424             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 425                                     &q_map_match);
 426             if (space_split && **map == *CHR_SPACE)
 427                 break;
 428
 429             /* add non-space char */
 430             memcpy(dst_term+j, s1, s0 - s1);
 431             j += (s0 - s1);
 432             if (!q_map_match)
 433             {
 434                 while (s1 < s0)
 435                 {
 436                     if (strchr(REGEX_CHARS, *s1))
 437                         dst[i++] = '\\';
 438                     dst[i++] = *s1++;
 439                 }
 440             }
 441             else
 442             {
 443                 char tmpbuf[80];
 444                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 445
 446                 strcpy(dst + i, map[0]);
 447                 i += strlen(map[0]);
 448             }
 449         }
 450     }
 451     dst[i] = '\0';
 452     dst_term[j++] = '\0';
 453     *src = s0;
 454     return i;
 455 }
 456
 457 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 458 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 459                     char *dst, int *errors, int space_split,
 460                     char *dst_term)
 461 {
 462     int i = 0;
 463     int j = 0;
 464     const char *s0;
 465     const char **map;
 466
 467     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 468         return 0;
 469     s0 = *src;
 470     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 471         isdigit(((const unsigned char *)s0)[1]))
 472     {
 473         *errors = s0[1] - '0';
 474         s0 += 3;
 475         if (*errors > 3)
 476             *errors = 3;
 477     }
 478     while (*s0)
 479     {
 480         if (strchr("^\\()[].*+?|-", *s0))
 481         {
 482             dst_term[j++] = *s0;
 483             dst[i++] = *s0++;
 484         }
 485         else
 486         {
 487             const char *s1 = s0;
 488             int q_map_match = 0;
 489             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 490                                     &q_map_match);
 491             if (space_split && **map == *CHR_SPACE)
 492                 break;
 493
 494             /* add non-space char */
 495             memcpy(dst_term+j, s1, s0 - s1);
 496             j += (s0 - s1);
 497             if (!q_map_match)
 498             {
 499                 while (s1 < s0)
 500                 {
 501                     if (strchr(REGEX_CHARS, *s1))
 502                         dst[i++] = '\\';
 503                     dst[i++] = *s1++;
 504                 }
 505             }
 506             else
 507             {
 508                 char tmpbuf[80];
 509                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 510
 511                 strcpy(dst + i, map[0]);
 512                 i += strlen(map[0]);
 513             }
 514         }
 515     }
 516     dst[i] = '\0';
 517     dst_term[j] = '\0';
 518     *src = s0;
 519
 520     return i;
 521 }
 522
 523 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 524 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 525                     char *dst, int space_split, char *dst_term)
 526 {
 527     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 528                     dst_term);
 529 }
 530
 531
 532 /* term_104: handle term, where trunc = Process # and ! */
 533 static int term_104(ZebraMaps zebra_maps, int reg_type,
 534                     const char **src, char *dst, int space_split,
 535                     char *dst_term)
 536 {
 537     const char *s0, *s1;
 538     const char **map;
 539     int i = 0;
 540     int j = 0;
 541
 542     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 543         return 0;
 544     s0 = *src;
 545     while (*s0)
 546     {
 547         if (*s0 == '?')
 548         {
 549             dst_term[j++] = *s0++;
 550             if (*s0 >= '0' && *s0 <= '9')
 551             {
 552                 int limit = 0;
 553                 while (*s0 >= '0' && *s0 <= '9')
 554                 {
 555                     limit = limit * 10 + (*s0 - '0');
 556                     dst_term[j++] = *s0++;
 557                 }
 558                 if (limit > 20)
 559                     limit = 20;
 560                 while (--limit >= 0)
 561                 {
 562                     dst[i++] = '.';
 563                     dst[i++] = '?';
 564                 }
 565             }
 566             else
 567             {
 568                 dst[i++] = '.';
 569                 dst[i++] = '*';
 570             }
 571         }
 572         else if (*s0 == '*')
 573         {
 574             dst[i++] = '.';
 575             dst[i++] = '*';
 576             dst_term[j++] = *s0++;
 577         }
 578         else if (*s0 == '#')
 579         {
 580             dst[i++] = '.';
 581             dst_term[j++] = *s0++;
 582         }
 583         else
 584         {
 585             s1 = s0;
 586             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 587             if (space_split && **map == *CHR_SPACE)
 588                 break;
 589             while (s1 < s0)
 590             {
 591                 if (strchr(REGEX_CHARS, *s1))
 592                     dst[i++] = '\\';
 593                 dst_term[j++] = *s1;
 594                 dst[i++] = *s1++;
 595             }
 596         }
 597     }
 598     dst[i] = '\0';
 599     dst_term[j++] = '\0';
 600     *src = s0;
 601     return i;
 602 }
 603
 604 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 605 static int term_105(ZebraMaps zebra_maps, int reg_type,
 606                     const char **src, char *dst, int space_split,
 607                     char *dst_term, int right_truncate)
 608 {
 609     const char *s0, *s1;
 610     const char **map;
 611     int i = 0;
 612     int j = 0;
 613
 614     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 615         return 0;
 616     s0 = *src;
 617     while (*s0)
 618     {
 619         if (*s0 == '*')
 620         {
 621             dst[i++] = '.';
 622             dst[i++] = '*';
 623             dst_term[j++] = *s0++;
 624         }
 625         else if (*s0 == '!')
 626         {
 627             dst[i++] = '.';
 628             dst_term[j++] = *s0++;
 629         }
 630         else
 631         {
 632             s1 = s0;
 633             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 634             if (space_split && **map == *CHR_SPACE)
 635                 break;
 636             while (s1 < s0)
 637             {
 638                 if (strchr(REGEX_CHARS, *s1))
 639                     dst[i++] = '\\';
 640                 dst_term[j++] = *s1;
 641                 dst[i++] = *s1++;
 642             }
 643         }
 644     }
 645     if (right_truncate)
 646     {
 647         dst[i++] = '.';
 648         dst[i++] = '*';
 649     }
 650     dst[i] = '\0';
 651
 652     dst_term[j++] = '\0';
 653     *src = s0;
 654     return i;
 655 }
 656
 657
 658 /* gen_regular_rel - generate regular expression from relation
 659  *  val:     border value (inclusive)
 660  *  islt:    1 if <=; 0 if >=.
 661  */
 662 static void gen_regular_rel(char *dst, int val, int islt)
 663 {
 664     int dst_p;
 665     int w, d, i;
 666     int pos = 0;
 667     char numstr[20];
 668
 669     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 670     if (val >= 0)
 671     {
 672         if (islt)
 673             strcpy(dst, "(-[0-9]+|(");
 674         else
 675             strcpy(dst, "((");
 676     }
 677     else
 678     {
 679         if (!islt)
 680         {
 681             strcpy(dst, "([0-9]+|-(");
 682             dst_p = strlen(dst);
 683             islt = 1;
 684         }
 685         else
 686         {
 687             strcpy(dst, "(-(");
 688             islt = 0;
 689         }
 690         val = -val;
 691     }
 692     dst_p = strlen(dst);
 693     sprintf(numstr, "%d", val);
 694     for (w = strlen(numstr); --w >= 0; pos++)
 695     {
 696         d = numstr[w];
 697         if (pos > 0)
 698         {
 699             if (islt)
 700             {
 701                 if (d == '0')
 702                     continue;
 703                 d--;
 704             }
 705             else
 706             {
 707                 if (d == '9')
 708                     continue;
 709                 d++;
 710             }
 711         }
 712
 713         strcpy(dst + dst_p, numstr);
 714         dst_p = strlen(dst) - pos - 1;
 715
 716         if (islt)
 717         {
 718             if (d != '0')
 719             {
 720                 dst[dst_p++] = '[';
 721                 dst[dst_p++] = '0';
 722                 dst[dst_p++] = '-';
 723                 dst[dst_p++] = d;
 724                 dst[dst_p++] = ']';
 725             }
 726             else
 727                 dst[dst_p++] = d;
 728         }
 729         else
 730         {
 731             if (d != '9')
 732             {
 733                 dst[dst_p++] = '[';
 734                 dst[dst_p++] = d;
 735                 dst[dst_p++] = '-';
 736                 dst[dst_p++] = '9';
 737                 dst[dst_p++] = ']';
 738             }
 739             else
 740                 dst[dst_p++] = d;
 741         }
 742         for (i = 0; i<pos; i++)
 743         {
 744             dst[dst_p++] = '[';
 745             dst[dst_p++] = '0';
 746             dst[dst_p++] = '-';
 747             dst[dst_p++] = '9';
 748             dst[dst_p++] = ']';
 749         }
 750         dst[dst_p++] = '|';
 751     }
 752     dst[dst_p] = '\0';
 753     if (islt)
 754     {
 755         /* match everything less than 10^(pos-1) */
 756         strcat(dst, "0*");
 757         for (i = 1; i<pos; i++)
 758             strcat(dst, "[0-9]?");
 759     }
 760     else
 761     {
 762         /* match everything greater than 10^pos */
 763         for (i = 0; i <= pos; i++)
 764             strcat(dst, "[0-9]");
 765         strcat(dst, "[0-9]*");
 766     }
 767     strcat(dst, "))");
 768 }
 769
 770 void string_rel_add_char(char **term_p, const char *src, int *indx)
 771 {
 772     if (src[*indx] == '\\')
 773         *(*term_p)++ = src[(*indx)++];
 774     *(*term_p)++ = src[(*indx)++];
 775 }
 776
 777 /*
 778  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 779  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 780  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 781  *              ([^-a].*|a[^-b].*|ab[c-].*)
 782  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 783  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 784  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 785  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 786  */
 787 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 788                            const char **term_sub, char *term_dict,
 789                            oid_value attributeSet,
 790                            int reg_type, int space_split, char *term_dst,
 791                            int *error_code)
 792 {
 793     AttrType relation;
 794     int relation_value;
 795     int i;
 796     char *term_tmp = term_dict + strlen(term_dict);
 797     char term_component[2*IT_MAX_WORD+20];
 798
 799     attr_init(&relation, zapt, 2);
 800     relation_value = attr_find(&relation, NULL);
 801
 802     *error_code = 0;
 803     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 804     switch (relation_value)
 805     {
 806     case 1:
 807         if (!term_100(zh->reg->zebra_maps, reg_type,
 808                       term_sub, term_component,
 809                       space_split, term_dst))
 810             return 0;
 811         yaz_log(log_level_rpn, "Relation <");
 812
 813         *term_tmp++ = '(';
 814         for (i = 0; term_component[i]; )
 815         {
 816             int j = 0;
 817
 818             if (i)
 819                 *term_tmp++ = '|';
 820             while (j < i)
 821                 string_rel_add_char(&term_tmp, term_component, &j);
 822
 823             *term_tmp++ = '[';
 824
 825             *term_tmp++ = '^';
 826             string_rel_add_char(&term_tmp, term_component, &i);
 827             *term_tmp++ = '-';
 828
 829             *term_tmp++ = ']';
 830             *term_tmp++ = '.';
 831             *term_tmp++ = '*';
 832
 833             if ((term_tmp - term_dict) > IT_MAX_WORD)
 834                 break;
 835         }
 836         *term_tmp++ = ')';
 837         *term_tmp = '\0';
 838         break;
 839     case 2:
 840         if (!term_100(zh->reg->zebra_maps, reg_type,
 841                       term_sub, term_component,
 842                       space_split, term_dst))
 843             return 0;
 844         yaz_log(log_level_rpn, "Relation <=");
 845
 846         *term_tmp++ = '(';
 847         for (i = 0; term_component[i]; )
 848         {
 849             int j = 0;
 850
 851             while (j < i)
 852                 string_rel_add_char(&term_tmp, term_component, &j);
 853             *term_tmp++ = '[';
 854
 855             *term_tmp++ = '^';
 856             string_rel_add_char(&term_tmp, term_component, &i);
 857             *term_tmp++ = '-';
 858
 859             *term_tmp++ = ']';
 860             *term_tmp++ = '.';
 861             *term_tmp++ = '*';
 862
 863             *term_tmp++ = '|';
 864
 865             if ((term_tmp - term_dict) > IT_MAX_WORD)
 866                 break;
 867         }
 868         for (i = 0; term_component[i]; )
 869             string_rel_add_char(&term_tmp, term_component, &i);
 870         *term_tmp++ = ')';
 871         *term_tmp = '\0';
 872         break;
 873     case 5:
 874         if (!term_100 (zh->reg->zebra_maps, reg_type,
 875                        term_sub, term_component, space_split, term_dst))
 876             return 0;
 877         yaz_log(log_level_rpn, "Relation >");
 878
 879         *term_tmp++ = '(';
 880         for (i = 0; term_component[i];)
 881         {
 882             int j = 0;
 883
 884             while (j < i)
 885                 string_rel_add_char(&term_tmp, term_component, &j);
 886             *term_tmp++ = '[';
 887
 888             *term_tmp++ = '^';
 889             *term_tmp++ = '-';
 890             string_rel_add_char(&term_tmp, term_component, &i);
 891
 892             *term_tmp++ = ']';
 893             *term_tmp++ = '.';
 894             *term_tmp++ = '*';
 895
 896             *term_tmp++ = '|';
 897
 898             if ((term_tmp - term_dict) > IT_MAX_WORD)
 899                 break;
 900         }
 901         for (i = 0; term_component[i];)
 902             string_rel_add_char(&term_tmp, term_component, &i);
 903         *term_tmp++ = '.';
 904         *term_tmp++ = '+';
 905         *term_tmp++ = ')';
 906         *term_tmp = '\0';
 907         break;
 908     case 4:
 909         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 910                       term_component, space_split, term_dst))
 911             return 0;
 912         yaz_log(log_level_rpn, "Relation >=");
 913
 914         *term_tmp++ = '(';
 915         for (i = 0; term_component[i];)
 916         {
 917             int j = 0;
 918
 919             if (i)
 920                 *term_tmp++ = '|';
 921             while (j < i)
 922                 string_rel_add_char(&term_tmp, term_component, &j);
 923             *term_tmp++ = '[';
 924
 925             if (term_component[i+1])
 926             {
 927                 *term_tmp++ = '^';
 928                 *term_tmp++ = '-';
 929                 string_rel_add_char(&term_tmp, term_component, &i);
 930             }
 931             else
 932             {
 933                 string_rel_add_char(&term_tmp, term_component, &i);
 934                 *term_tmp++ = '-';
 935             }
 936             *term_tmp++ = ']';
 937             *term_tmp++ = '.';
 938             *term_tmp++ = '*';
 939
 940             if ((term_tmp - term_dict) > IT_MAX_WORD)
 941                 break;
 942         }
 943         *term_tmp++ = ')';
 944         *term_tmp = '\0';
 945         break;
 946     case 3:
 947     case 102:
 948     case -1:
 949         yaz_log(log_level_rpn, "Relation =");
 950         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 951                       term_component, space_split, term_dst))
 952             return 0;
 953         strcat(term_tmp, "(");
 954         strcat(term_tmp, term_component);
 955         strcat(term_tmp, ")");
 956         break;
 957     default:
 958         *error_code = 117;
 959         return 0;
 960     }
 961     return 1;
 962 }
 963
 964 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 965                              const char **term_sub,
 966                              oid_value attributeSet, NMEM stream,
 967                              struct grep_info *grep_info,
 968                              int reg_type, int complete_flag,
 969                              int num_bases, char **basenames,
 970                              char *term_dst, int xpath_use);
 971
 972 static ZEBRA_RES term_trunc(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 973                             const char **term_sub,
 974                             oid_value attributeSet, NMEM stream,
 975                             struct grep_info *grep_info,
 976                             int reg_type, int complete_flag,
 977                             int num_bases, char **basenames,
 978                             char *term_dst,
 979                             const char *rank_type, int xpath_use,
 980                             NMEM rset_nmem,
 981                             RSET *rset)
 982 {
 983     ZEBRA_RES res;
 984     *rset = 0;
 985     grep_info->isam_p_indx = 0;
 986     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 987                       reg_type, complete_flag, num_bases, basenames,
 988                       term_dst, xpath_use);
 989     if (res != ZEBRA_OK)
 990         return res;
 991     if (!*term_sub)  /* no more terms ? */
 992         return res;
 993     yaz_log(log_level_rpn, "term: %s", term_dst);
 994     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 995                        grep_info->isam_p_indx, term_dst,
 996                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 997                        zapt->term->which, rset_nmem,
 998                        key_it_ctrl, key_it_ctrl->scope);
 999     if (!*rset)
1000         return ZEBRA_FAIL;
1001     return ZEBRA_OK;
1002 }
1003
1004 static char *nmem_strdup_i(NMEM nmem, int v)
1005 {
1006     char val_str[64];
1007     sprintf(val_str, "%d", v);
1008     return nmem_strdup(nmem, val_str);
1009 }
1010
1011 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1012                              const char **term_sub,
1013                              oid_value attributeSet, NMEM stream,
1014                              struct grep_info *grep_info,
1015                              int reg_type, int complete_flag,
1016                              int num_bases, char **basenames,
1017                              char *term_dst, int xpath_use)
1018 {
1019     char term_dict[2*IT_MAX_WORD+4000];
1020     int j, r, base_no;
1021     AttrType truncation;
1022     int truncation_value;
1023     AttrType use;
1024     int use_value;
1025     const char *use_string = 0;
1026     oid_value curAttributeSet = attributeSet;
1027     const char *termp;
1028     struct rpn_char_map_info rcmi;
1029     int space_split = complete_flag ? 0 : 1;
1030
1031     int bases_ok = 0;     /* no of databases with OK attribute */
1032     int errCode = 0;      /* err code (if any is not OK) */
1033     char *errString = 0;  /* addinfo */
1034
1035     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1036     attr_init(&use, zapt, 1);
1037     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1038     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1039     attr_init(&truncation, zapt, 5);
1040     truncation_value = attr_find(&truncation, NULL);
1041     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1042
1043     if (use_value == -1)    /* no attribute - assumy "any" */
1044         use_value = 1016;
1045     for (base_no = 0; base_no < num_bases; base_no++)
1046     {
1047         int ord = -1;
1048         int attr_ok = 0;
1049         int regex_range = 0;
1050         int init_pos = 0;
1051         attent attp;
1052         data1_local_attribute id_xpath_attr;
1053         data1_local_attribute *local_attr;
1054         int max_pos, prefix_len = 0;
1055         int relation_error;
1056
1057         termp = *term_sub;
1058
1059         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1060         {
1061             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
1062             zh->errString = basenames[base_no];
1063             return ZEBRA_FAIL;
1064         }
1065         if (xpath_use > 0 && use_value == -2)
1066         {
1067             /* xpath mode and we have a string attribute */
1068             attp.local_attributes = &id_xpath_attr;
1069             attp.attset_ordinal = VAL_IDXPATH;
1070             id_xpath_attr.next = 0;
1071
1072             use_value = xpath_use;  /* xpath_use as use-attribute now */
1073             id_xpath_attr.local = use_value;
1074         }
1075         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1076         {
1077             /* X-Path attribute, use numeric value directly */
1078             attp.local_attributes = &id_xpath_attr;
1079             attp.attset_ordinal = VAL_IDXPATH;
1080             id_xpath_attr.next = 0;
1081             id_xpath_attr.local = use_value;
1082         }
1083         else if (use_string &&
1084                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1085                                                      use_string)) >= 0)
1086         {
1087             /* we have a match for a raw string attribute */
1088             char ord_buf[32];
1089             int i, ord_len;
1090
1091             if (prefix_len)
1092                 term_dict[prefix_len++] = '|';
1093             else
1094                 term_dict[prefix_len++] = '(';
1095
1096             ord_len = key_SU_encode (ord, ord_buf);
1097             for (i = 0; i<ord_len; i++)
1098             {
1099                 term_dict[prefix_len++] = 1;
1100                 term_dict[prefix_len++] = ord_buf[i];
1101             }
1102             attp.local_attributes = 0;  /* no more attributes */
1103         }
1104         else
1105         {
1106             /* lookup in the .att files . Allow string as well */
1107             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1108                                       use_string)))
1109             {
1110                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1111                         curAttributeSet, use_value, r);
1112                 if (r == -1)
1113                 {
1114                     /* set was found, but value wasn't defined */
1115                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1116                     if (use_string)
1117                         errString = nmem_strdup(stream, use_string);
1118                     else
1119                         errString = nmem_strdup_i (stream, use_value);
1120                 }
1121                 else
1122                 {
1123                     int oid[OID_SIZE];
1124                     struct oident oident;
1125
1126                     oident.proto = PROTO_Z3950;
1127                     oident.oclass = CLASS_ATTSET;
1128                     oident.value = curAttributeSet;
1129                     oid_ent_to_oid (&oident, oid);
1130
1131                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
1132                     errString = nmem_strdup(stream, oident.desc);
1133                 }
1134                 continue;
1135             }
1136         }
1137         for (local_attr = attp.local_attributes; local_attr;
1138              local_attr = local_attr->next)
1139         {
1140             char ord_buf[32];
1141             int i, ord_len;
1142
1143             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1144                                               attp.attset_ordinal,
1145                                               local_attr->local);
1146             if (ord < 0)
1147                 continue;
1148             if (prefix_len)
1149                 term_dict[prefix_len++] = '|';
1150             else
1151                 term_dict[prefix_len++] = '(';
1152
1153             ord_len = key_SU_encode (ord, ord_buf);
1154             for (i = 0; i<ord_len; i++)
1155             {
1156                 term_dict[prefix_len++] = 1;
1157                 term_dict[prefix_len++] = ord_buf[i];
1158             }
1159         }
1160         bases_ok++;
1161         if (prefix_len)
1162             attr_ok = 1;
1163
1164         term_dict[prefix_len++] = ')';
1165         term_dict[prefix_len++] = 1;
1166         term_dict[prefix_len++] = reg_type;
1167         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1168         term_dict[prefix_len] = '\0';
1169         j = prefix_len;
1170         switch (truncation_value)
1171         {
1172         case -1:         /* not specified */
1173         case 100:        /* do not truncate */
1174             if (!string_relation (zh, zapt, &termp, term_dict,
1175                                   attributeSet,
1176                                   reg_type, space_split, term_dst,
1177                                   &relation_error))
1178             {
1179                 if (relation_error)
1180                 {
1181                     zh->errCode = relation_error;
1182                     return ZEBRA_FAIL;
1183                 }
1184                 *term_sub = 0;
1185                 return ZEBRA_OK;
1186             }
1187             break;
1188         case 1:          /* right truncation */
1189             term_dict[j++] = '(';
1190             if (!term_100(zh->reg->zebra_maps, reg_type,
1191                           &termp, term_dict + j, space_split, term_dst))
1192             {
1193                 *term_sub = 0;
1194                 return ZEBRA_OK;
1195             }
1196             strcat(term_dict, ".*)");
1197             break;
1198         case 2:          /* keft truncation */
1199             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1200             if (!term_100(zh->reg->zebra_maps, reg_type,
1201                           &termp, term_dict + j, space_split, term_dst))
1202             {
1203                 *term_sub = 0;
1204                 return ZEBRA_OK;
1205             }
1206             strcat(term_dict, ")");
1207             break;
1208         case 3:          /* left&right truncation */
1209             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1210             if (!term_100(zh->reg->zebra_maps, reg_type,
1211                           &termp, term_dict + j, space_split, term_dst))
1212             {
1213                 *term_sub = 0;
1214                 return ZEBRA_OK;
1215             }
1216             strcat(term_dict, ".*)");
1217             break;
1218         case 101:        /* process # in term */
1219             term_dict[j++] = '(';
1220             if (!term_101(zh->reg->zebra_maps, reg_type,
1221                           &termp, term_dict + j, space_split, term_dst))
1222             {
1223                 *term_sub = 0;
1224                 return ZEBRA_OK;
1225             }
1226             strcat(term_dict, ")");
1227             break;
1228         case 102:        /* Regexp-1 */
1229             term_dict[j++] = '(';
1230             if (!term_102(zh->reg->zebra_maps, reg_type,
1231                           &termp, term_dict + j, space_split, term_dst))
1232             {
1233                 *term_sub = 0;
1234                 return ZEBRA_OK;
1235             }
1236             strcat(term_dict, ")");
1237             break;
1238         case 103:       /* Regexp-2 */
1239             r = 1;
1240             term_dict[j++] = '(';
1241             init_pos = 2;
1242             if (!term_103(zh->reg->zebra_maps, reg_type,
1243                           &termp, term_dict + j, &regex_range,
1244                           space_split, term_dst))
1245             {
1246                 *term_sub = 0;
1247                 return ZEBRA_OK;
1248             }
1249             strcat(term_dict, ")");
1250         case 104:        /* process # and ! in term */
1251             term_dict[j++] = '(';
1252             if (!term_104(zh->reg->zebra_maps, reg_type,
1253                           &termp, term_dict + j, space_split, term_dst))
1254             {
1255                 *term_sub = 0;
1256                 return ZEBRA_OK;
1257             }
1258             strcat(term_dict, ")");
1259             break;
1260         case 105:        /* process * and ! in term */
1261             term_dict[j++] = '(';
1262             if (!term_105(zh->reg->zebra_maps, reg_type,
1263                           &termp, term_dict + j, space_split, term_dst, 1))
1264             {
1265                 *term_sub = 0;
1266                 return ZEBRA_OK;
1267             }
1268             strcat(term_dict, ")");
1269             break;
1270         case 106:        /* process * and ! in term */
1271             term_dict[j++] = '(';
1272             if (!term_105(zh->reg->zebra_maps, reg_type,
1273                           &termp, term_dict + j, space_split, term_dst, 0))
1274             {
1275                 *term_sub = 0;
1276                 return ZEBRA_OK;
1277             }
1278             strcat(term_dict, ")");
1279             break;
1280         default:
1281             zh->errCode = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1282             zh->errString = nmem_strdup_i(stream, truncation_value);
1283             return ZEBRA_FAIL;
1284         }
1285         if (attr_ok)
1286         {
1287             char buf[80];
1288             const char *input = term_dict + prefix_len;
1289             esc_str(buf, sizeof(buf), input, strlen(input));
1290         }
1291         if (attr_ok)
1292         {
1293             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1294             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1295                                  grep_info, &max_pos, init_pos,
1296                                  grep_handle);
1297             if (r)
1298                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1299         }
1300     }
1301     if (!bases_ok)
1302     {
1303         zh->errCode = errCode;
1304         zh->errString = errString;
1305         return ZEBRA_FAIL;
1306     }
1307     *term_sub = termp;
1308     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1309     return ZEBRA_OK;
1310 }
1311
1312
1313 /* convert APT search term to UTF8 */
1314 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1315                                    char *termz)
1316 {
1317     size_t sizez;
1318     Z_Term *term = zapt->term;
1319
1320     switch (term->which)
1321     {
1322     case Z_Term_general:
1323         if (zh->iconv_to_utf8 != 0)
1324         {
1325             char *inbuf = term->u.general->buf;
1326             size_t inleft = term->u.general->len;
1327             char *outbuf = termz;
1328             size_t outleft = IT_MAX_WORD-1;
1329             size_t ret;
1330
1331             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1332                         &outbuf, &outleft);
1333             if (ret == (size_t)(-1))
1334             {
1335                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1336                 zh->errCode =
1337                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_;
1338                 return -1;
1339             }
1340             *outbuf = 0;
1341         }
1342         else
1343         {
1344             sizez = term->u.general->len;
1345             if (sizez > IT_MAX_WORD-1)
1346                 sizez = IT_MAX_WORD-1;
1347             memcpy (termz, term->u.general->buf, sizez);
1348             termz[sizez] = '\0';
1349         }
1350         break;
1351     case Z_Term_characterString:
1352         sizez = strlen(term->u.characterString);
1353         if (sizez > IT_MAX_WORD-1)
1354             sizez = IT_MAX_WORD-1;
1355         memcpy (termz, term->u.characterString, sizez);
1356         termz[sizez] = '\0';
1357         break;
1358     default:
1359         zh->errCode = YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM;
1360         return ZEBRA_FAIL;
1361     }
1362     return ZEBRA_OK;
1363 }
1364
1365 /* convert APT SCAN term to internal cmap */
1366 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1367                                  char *termz, int reg_type)
1368 {
1369     char termz0[IT_MAX_WORD];
1370
1371     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1372         return ZEBRA_FAIL;    /* error */
1373     else
1374     {
1375         const char **map;
1376         const char *cp = (const char *) termz0;
1377         const char *cp_end = cp + strlen(cp);
1378         const char *src;
1379         int i = 0;
1380         const char *space_map = NULL;
1381         int len;
1382
1383         while ((len = (cp_end - cp)) > 0)
1384         {
1385             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1386             if (**map == *CHR_SPACE)
1387                 space_map = *map;
1388             else
1389             {
1390                 if (i && space_map)
1391                     for (src = space_map; *src; src++)
1392                         termz[i++] = *src;
1393                 space_map = NULL;
1394                 for (src = *map; *src; src++)
1395                     termz[i++] = *src;
1396             }
1397         }
1398         termz[i] = '\0';
1399     }
1400     return ZEBRA_OK;
1401 }
1402
1403 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1404                      const char *termz, NMEM stream, unsigned reg_id)
1405 {
1406     WRBUF wrbuf = 0;
1407     AttrType truncation;
1408     int truncation_value;
1409     char *ex_list = 0;
1410
1411     attr_init(&truncation, zapt, 5);
1412     truncation_value = attr_find(&truncation, NULL);
1413
1414     switch (truncation_value)
1415     {
1416     default:
1417         ex_list = "";
1418         break;
1419     case 101:
1420         ex_list = "#";
1421         break;
1422     case 102:
1423     case 103:
1424         ex_list = 0;
1425         break;
1426     case 104:
1427         ex_list = "!#";
1428         break;
1429     case 105:
1430         ex_list = "!*";
1431         break;
1432     }
1433     if (ex_list)
1434         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1435                               termz, strlen(termz));
1436     if (!wrbuf)
1437         return nmem_strdup(stream, termz);
1438     else
1439     {
1440         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1441         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1442         buf[wrbuf_len(wrbuf)] = '\0';
1443         return buf;
1444     }
1445 }
1446
1447 static void grep_info_delete(struct grep_info *grep_info)
1448 {
1449 #ifdef TERM_COUNT
1450     xfree(grep_info->term_no);
1451 #endif
1452     xfree(grep_info->isam_p_buf);
1453 }
1454
1455 static int grep_info_prepare(ZebraHandle zh,
1456                              Z_AttributesPlusTerm *zapt,
1457                              struct grep_info *grep_info,
1458                              int reg_type,
1459                              NMEM stream)
1460 {
1461     AttrType termset;
1462     int termset_value_numeric;
1463     const char *termset_value_string;
1464
1465 #ifdef TERM_COUNT
1466     grep_info->term_no = 0;
1467 #endif
1468     grep_info->isam_p_size = 0;
1469     grep_info->isam_p_buf = NULL;
1470     grep_info->zh = zh;
1471     grep_info->reg_type = reg_type;
1472     grep_info->termset = 0;
1473
1474     if (!zapt)
1475         return 0;
1476     attr_init(&termset, zapt, 8);
1477     termset_value_numeric =
1478         attr_find_ex(&termset, NULL, &termset_value_string);
1479     if (termset_value_numeric != -1)
1480     {
1481         char resname[32];
1482         const char *termset_name = 0;
1483         if (termset_value_numeric != -2)
1484         {
1485
1486             sprintf(resname, "%d", termset_value_numeric);
1487             termset_name = resname;
1488         }
1489         else
1490             termset_name = termset_value_string;
1491         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1492         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1493         if (!grep_info->termset)
1494         {
1495             zh->errCode = YAZ_BIB1_ILLEGAL_RESULT_SET_NAME;
1496             zh->errString = nmem_strdup(stream, termset_name);
1497             return -1;
1498         }
1499     }
1500     return 0;
1501 }
1502
1503
1504 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1505                                  Z_AttributesPlusTerm *zapt,
1506                                  const char *termz_org,
1507                                  oid_value attributeSet,
1508                                  NMEM stream,
1509                                  int reg_type, int complete_flag,
1510                                  const char *rank_type, int xpath_use,
1511                                  int num_bases, char **basenames,
1512                                  NMEM rset_nmem,
1513                                  RSET **result_sets, int *num_result_sets)
1514 {
1515     char term_dst[IT_MAX_WORD+1];
1516     struct grep_info grep_info;
1517     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1518     const char *termp = termz;
1519     int alloc_sets = 0;
1520
1521     *num_result_sets = 0;
1522     *term_dst = 0;
1523     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1524         return ZEBRA_FAIL;
1525     while(1)
1526     {
1527         ZEBRA_RES res;
1528
1529         if (alloc_sets == *num_result_sets)
1530         {
1531             int add = 10;
1532             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1533                                               sizeof(*rnew));
1534             if (alloc_sets)
1535                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1536             alloc_sets = alloc_sets + add;
1537             *result_sets = rnew;
1538         }
1539         res = term_trunc(zh, zapt, &termp, attributeSet,
1540                          stream, &grep_info,
1541                          reg_type, complete_flag,
1542                          num_bases, basenames,
1543                          term_dst, rank_type,
1544                          xpath_use, rset_nmem,
1545                          &(*result_sets)[*num_result_sets]);
1546         if (res != ZEBRA_OK)
1547         {
1548             int i;
1549             for (i = 0; i < *num_result_sets; i++)
1550                 rset_delete((*result_sets)[i]);
1551             grep_info_delete (&grep_info);
1552             return res;
1553         }
1554         if ((*result_sets)[*num_result_sets] == 0)
1555             break;
1556         (*num_result_sets)++;
1557     }
1558     grep_info_delete(&grep_info);
1559     return ZEBRA_OK;
1560 }
1561
1562 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1563                                        Z_AttributesPlusTerm *zapt,
1564                                        const char *termz_org,
1565                                        oid_value attributeSet,
1566                                        NMEM stream,
1567                                        int reg_type, int complete_flag,
1568                                        const char *rank_type, int xpath_use,
1569                                        int num_bases, char **basenames,
1570                                        NMEM rset_nmem,
1571                                        RSET *rset)
1572 {
1573     RSET *result_sets = 0;
1574     int num_result_sets = 0;
1575     ZEBRA_RES res =
1576         term_list_trunc(zh, zapt, termz_org, attributeSet,
1577                         stream, reg_type, complete_flag,
1578                         rank_type, xpath_use,
1579                         num_bases, basenames,
1580                         rset_nmem,
1581                         &result_sets, &num_result_sets);
1582     if (res != ZEBRA_OK)
1583         return res;
1584     if (num_result_sets == 0)
1585         *rset = rsnull_create (rset_nmem, key_it_ctrl);
1586     else if (num_result_sets == 1)
1587         *rset = result_sets[0];
1588     else
1589         *rset = rsprox_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1590                               num_result_sets, result_sets,
1591                               1 /* ordered */, 0 /* exclusion */,
1592                               3 /* relation */, 1 /* distance */);
1593     if (!*rset)
1594         return ZEBRA_FAIL;
1595     return ZEBRA_OK;
1596 }
1597
1598 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1599                                         Z_AttributesPlusTerm *zapt,
1600                                         const char *termz_org,
1601                                         oid_value attributeSet,
1602                                         NMEM stream,
1603                                         int reg_type, int complete_flag,
1604                                         const char *rank_type,
1605                                         int xpath_use,
1606                                         int num_bases, char **basenames,
1607                                         NMEM rset_nmem,
1608                                         RSET *rset)
1609 {
1610     RSET *result_sets = 0;
1611     int num_result_sets = 0;
1612     ZEBRA_RES res =
1613         term_list_trunc(zh, zapt, termz_org, attributeSet,
1614                         stream, reg_type, complete_flag,
1615                         rank_type, xpath_use,
1616                         num_bases, basenames,
1617                         rset_nmem,
1618                         &result_sets, &num_result_sets);
1619     if (res != ZEBRA_OK)
1620         return res;
1621     if (num_result_sets == 0)
1622         *rset = rsnull_create (rset_nmem, key_it_ctrl);
1623     else if (num_result_sets == 1)
1624         *rset = result_sets[0];
1625     else
1626         *rset = rsmulti_or_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1627                                   num_result_sets, result_sets);
1628     if (!*rset)
1629         return ZEBRA_FAIL;
1630     return ZEBRA_OK;
1631 }
1632
1633 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1634                                          Z_AttributesPlusTerm *zapt,
1635                                          const char *termz_org,
1636                                          oid_value attributeSet,
1637                                          NMEM stream,
1638                                          int reg_type, int complete_flag,
1639                                          const char *rank_type,
1640                                          int xpath_use,
1641                                          int num_bases, char **basenames,
1642                                          NMEM rset_nmem,
1643                                          RSET *rset)
1644 {
1645     RSET *result_sets = 0;
1646     int num_result_sets = 0;
1647     ZEBRA_RES res =
1648         term_list_trunc(zh, zapt, termz_org, attributeSet,
1649                         stream, reg_type, complete_flag,
1650                         rank_type, xpath_use,
1651                         num_bases, basenames,
1652                         rset_nmem,
1653                         &result_sets, &num_result_sets);
1654     if (res != ZEBRA_OK)
1655         return res;
1656     if (num_result_sets == 0)
1657         *rset = rsnull_create (rset_nmem, key_it_ctrl);
1658     else if (num_result_sets == 1)
1659         *rset = result_sets[0];
1660     else
1661         *rset = rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1662                                    num_result_sets, result_sets);
1663     if (!*rset)
1664         return ZEBRA_FAIL;
1665     return ZEBRA_OK;
1666 }
1667
1668 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1669                             const char **term_sub,
1670                             char *term_dict,
1671                             oid_value attributeSet,
1672                             struct grep_info *grep_info,
1673                             int *max_pos,
1674                             int reg_type,
1675                             char *term_dst,
1676                             int *error_code)
1677 {
1678     AttrType relation;
1679     int relation_value;
1680     int term_value;
1681     int r;
1682     char *term_tmp = term_dict + strlen(term_dict);
1683
1684     *error_code = 0;
1685     attr_init(&relation, zapt, 2);
1686     relation_value = attr_find(&relation, NULL);
1687
1688     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1689
1690     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1691                   term_dst))
1692         return 0;
1693     term_value = atoi (term_tmp);
1694     switch (relation_value)
1695     {
1696     case 1:
1697         yaz_log(log_level_rpn, "Relation <");
1698         gen_regular_rel(term_tmp, term_value-1, 1);
1699         break;
1700     case 2:
1701         yaz_log(log_level_rpn, "Relation <=");
1702         gen_regular_rel(term_tmp, term_value, 1);
1703         break;
1704     case 4:
1705         yaz_log(log_level_rpn, "Relation >=");
1706         gen_regular_rel(term_tmp, term_value, 0);
1707         break;
1708     case 5:
1709         yaz_log(log_level_rpn, "Relation >");
1710         gen_regular_rel(term_tmp, term_value+1, 0);
1711         break;
1712     case -1:
1713     case 3:
1714         yaz_log(log_level_rpn, "Relation =");
1715         sprintf(term_tmp, "(0*%d)", term_value);
1716         break;
1717     default:
1718         *error_code = 117;
1719         return 0;
1720     }
1721     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1722     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1723                           0, grep_handle);
1724     if (r)
1725         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1726     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1727     return 1;
1728 }
1729
1730 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1731                               const char **term_sub,
1732                               oid_value attributeSet,
1733                               struct grep_info *grep_info,
1734                               int reg_type, int complete_flag,
1735                               int num_bases, char **basenames,
1736                               char *term_dst, int xpath_use, NMEM stream)
1737 {
1738     char term_dict[2*IT_MAX_WORD+2];
1739     int r, base_no;
1740     AttrType use;
1741     int use_value;
1742     const char *use_string = 0;
1743     oid_value curAttributeSet = attributeSet;
1744     const char *termp;
1745     struct rpn_char_map_info rcmi;
1746
1747     int bases_ok = 0;     /* no of databases with OK attribute */
1748     int errCode = 0;      /* err code (if any is not OK) */
1749     char *errString = 0;  /* addinfo */
1750
1751     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1752     attr_init(&use, zapt, 1);
1753     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1754
1755     if (use_value == -1)
1756         use_value = 1016;
1757
1758     for (base_no = 0; base_no < num_bases; base_no++)
1759     {
1760         attent attp;
1761         data1_local_attribute id_xpath_attr;
1762         data1_local_attribute *local_attr;
1763         int max_pos, prefix_len = 0;
1764         int relation_error = 0;
1765
1766         termp = *term_sub;
1767         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1768         {
1769             use_value = xpath_use;
1770             attp.local_attributes = &id_xpath_attr;
1771             attp.attset_ordinal = VAL_IDXPATH;
1772             id_xpath_attr.next = 0;
1773             id_xpath_attr.local = use_value;
1774         }
1775         else if (curAttributeSet == VAL_IDXPATH)
1776         {
1777             attp.local_attributes = &id_xpath_attr;
1778             attp.attset_ordinal = VAL_IDXPATH;
1779             id_xpath_attr.next = 0;
1780             id_xpath_attr.local = use_value;
1781         }
1782         else
1783         {
1784             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1785                                             use_string)))
1786             {
1787                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1788                       curAttributeSet, use_value, r);
1789                 if (r == -1)
1790                 {
1791                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1792                     if (use_string)
1793                         errString = nmem_strdup(stream, use_string);
1794                     else
1795                         errString = nmem_strdup_i (stream, use_value);
1796                 }
1797                 else
1798                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
1799                 continue;
1800             }
1801         }
1802         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1803         {
1804             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
1805             zh->errString = basenames[base_no];
1806             return -1;
1807         }
1808         for (local_attr = attp.local_attributes; local_attr;
1809              local_attr = local_attr->next)
1810         {
1811             int ord;
1812             char ord_buf[32];
1813             int i, ord_len;
1814
1815             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1816                                               attp.attset_ordinal,
1817                                               local_attr->local);
1818             if (ord < 0)
1819                 continue;
1820             if (prefix_len)
1821                 term_dict[prefix_len++] = '|';
1822             else
1823                 term_dict[prefix_len++] = '(';
1824
1825             ord_len = key_SU_encode (ord, ord_buf);
1826             for (i = 0; i<ord_len; i++)
1827             {
1828                 term_dict[prefix_len++] = 1;
1829                 term_dict[prefix_len++] = ord_buf[i];
1830             }
1831         }
1832         if (!prefix_len)
1833         {
1834             errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1835             errString = nmem_strdup_i(stream, use_value);
1836             continue;
1837         }
1838         bases_ok++;
1839         term_dict[prefix_len++] = ')';
1840         term_dict[prefix_len++] = 1;
1841         term_dict[prefix_len++] = reg_type;
1842         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1843         term_dict[prefix_len] = '\0';
1844         if (!numeric_relation(zh, zapt, &termp, term_dict,
1845                               attributeSet, grep_info, &max_pos, reg_type,
1846                               term_dst, &relation_error))
1847         {
1848             if (relation_error)
1849             {
1850                 zh->errCode = relation_error;
1851                 zh->errString = 0;
1852                 return ZEBRA_FAIL;
1853             }
1854             *term_sub = 0;
1855             return ZEBRA_OK;
1856         }
1857     }
1858     if (!bases_ok)
1859     {
1860         zh->errCode = errCode;
1861         zh->errString = errString;
1862         return ZEBRA_FAIL;
1863     }
1864     *term_sub = termp;
1865     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1866     return ZEBRA_OK;
1867 }
1868
1869 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1870                                         Z_AttributesPlusTerm *zapt,
1871                                         const char *termz,
1872                                         oid_value attributeSet,
1873                                         NMEM stream,
1874                                         int reg_type, int complete_flag,
1875                                         const char *rank_type, int xpath_use,
1876                                         int num_bases, char **basenames,
1877                                         NMEM rset_nmem,
1878                                         RSET *rset)
1879 {
1880     char term_dst[IT_MAX_WORD+1];
1881     const char *termp = termz;
1882     RSET *result_sets = 0;
1883     int num_result_sets = 0;
1884     ZEBRA_RES res;
1885     struct grep_info grep_info;
1886     int alloc_sets = 0;
1887
1888     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1889     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1890         return ZEBRA_FAIL;
1891     while (1)
1892     {
1893         if (alloc_sets == num_result_sets)
1894         {
1895             int add = 10;
1896             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1897                                               sizeof(*rnew));
1898             if (alloc_sets)
1899                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1900             alloc_sets = alloc_sets + add;
1901             result_sets = rnew;
1902         }
1903         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1904         grep_info.isam_p_indx = 0;
1905         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1906                            reg_type, complete_flag, num_bases, basenames,
1907                            term_dst, xpath_use,
1908                            stream);
1909         if (res == ZEBRA_FAIL || termp == 0)
1910             break;
1911         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1912         result_sets[num_result_sets] =
1913             rset_trunc(zh, grep_info.isam_p_buf,
1914                        grep_info.isam_p_indx, term_dst,
1915                        strlen(term_dst), rank_type,
1916                        0 /* preserve position */,
1917                        zapt->term->which, rset_nmem,
1918                        key_it_ctrl,key_it_ctrl->scope);
1919         if (!result_sets[num_result_sets])
1920             break;
1921         num_result_sets++;
1922     }
1923     grep_info_delete(&grep_info);
1924     if (termp)
1925     {
1926         int i;
1927         for (i = 0; i<num_result_sets; i++)
1928             rset_delete(result_sets[i]);
1929         return ZEBRA_FAIL;
1930     }
1931     if (num_result_sets == 0)
1932         *rset = rsnull_create(rset_nmem, key_it_ctrl);
1933     if (num_result_sets == 1)
1934         *rset = result_sets[0];
1935     else
1936         *rset = rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1937                                    num_result_sets, result_sets);
1938     if (!*rset)
1939         return ZEBRA_FAIL;
1940     return ZEBRA_OK;
1941 }
1942
1943 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1944                                       Z_AttributesPlusTerm *zapt,
1945                                       const char *termz,
1946                                       oid_value attributeSet,
1947                                       NMEM stream,
1948                                       const char *rank_type, NMEM rset_nmem,
1949                                       RSET *rset)
1950 {
1951     RSFD rsfd;
1952     struct it_key key;
1953     int sys;
1954     *rset = rstemp_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1955                           res_get (zh->res, "setTmpDir"),0 );
1956     rsfd = rset_open(*rset, RSETF_WRITE);
1957
1958     sys = atoi(termz);
1959     if (sys <= 0)
1960         sys = 1;
1961     key.mem[0] = sys;
1962     key.mem[1] = 1;
1963     key.len = 2;
1964     rset_write (rsfd, &key);
1965     rset_close (rsfd);
1966     return ZEBRA_OK;
1967 }
1968
1969 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1970                                oid_value attributeSet, NMEM stream,
1971                                Z_SortKeySpecList *sort_sequence,
1972                                const char *rank_type,
1973                                RSET *rset)
1974 {
1975     int i;
1976     int sort_relation_value;
1977     AttrType sort_relation_type;
1978     int use_value;
1979     AttrType use_type;
1980     Z_SortKeySpec *sks;
1981     Z_SortKey *sk;
1982     Z_AttributeElement *ae;
1983     int oid[OID_SIZE];
1984     oident oe;
1985     char termz[20];
1986
1987     attr_init(&sort_relation_type, zapt, 7);
1988     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1989
1990     attr_init(&use_type, zapt, 1);
1991     use_value = attr_find(&use_type, &attributeSet);
1992
1993     if (!sort_sequence->specs)
1994     {
1995         sort_sequence->num_specs = 10;
1996         sort_sequence->specs = (Z_SortKeySpec **)
1997             nmem_malloc(stream, sort_sequence->num_specs *
1998                          sizeof(*sort_sequence->specs));
1999         for (i = 0; i<sort_sequence->num_specs; i++)
2000             sort_sequence->specs[i] = 0;
2001     }
2002     if (zapt->term->which != Z_Term_general)
2003         i = 0;
2004     else
2005         i = atoi_n ((char *) zapt->term->u.general->buf,
2006                     zapt->term->u.general->len);
2007     if (i >= sort_sequence->num_specs)
2008         i = 0;
2009     sprintf(termz, "%d", i);
2010
2011     oe.proto = PROTO_Z3950;
2012     oe.oclass = CLASS_ATTSET;
2013     oe.value = attributeSet;
2014     if (!oid_ent_to_oid (&oe, oid))
2015         return ZEBRA_FAIL;
2016
2017     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2018     sks->sortElement = (Z_SortElement *)
2019         nmem_malloc(stream, sizeof(*sks->sortElement));
2020     sks->sortElement->which = Z_SortElement_generic;
2021     sk = sks->sortElement->u.generic = (Z_SortKey *)
2022         nmem_malloc(stream, sizeof(*sk));
2023     sk->which = Z_SortKey_sortAttributes;
2024     sk->u.sortAttributes = (Z_SortAttributes *)
2025         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2026
2027     sk->u.sortAttributes->id = oid;
2028     sk->u.sortAttributes->list = (Z_AttributeList *)
2029         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
2030     sk->u.sortAttributes->list->num_attributes = 1;
2031     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
2032         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
2033     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
2034         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
2035     ae->attributeSet = 0;
2036     ae->attributeType = (int *)
2037         nmem_malloc(stream, sizeof(*ae->attributeType));
2038     *ae->attributeType = 1;
2039     ae->which = Z_AttributeValue_numeric;
2040     ae->value.numeric = (int *)
2041         nmem_malloc(stream, sizeof(*ae->value.numeric));
2042     *ae->value.numeric = use_value;
2043
2044     sks->sortRelation = (int *)
2045         nmem_malloc(stream, sizeof(*sks->sortRelation));
2046     if (sort_relation_value == 1)
2047         *sks->sortRelation = Z_SortKeySpec_ascending;
2048     else if (sort_relation_value == 2)
2049         *sks->sortRelation = Z_SortKeySpec_descending;
2050     else
2051         *sks->sortRelation = Z_SortKeySpec_ascending;
2052
2053     sks->caseSensitivity = (int *)
2054         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2055     *sks->caseSensitivity = 0;
2056
2057     sks->which = Z_SortKeySpec_null;
2058     sks->u.null = odr_nullval ();
2059     sort_sequence->specs[i] = sks;
2060     *rset = rsnull_create (NULL, key_it_ctrl);
2061     return ZEBRA_OK;
2062 }
2063
2064
2065 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2066                        oid_value attributeSet,
2067                        struct xpath_location_step *xpath, int max, NMEM mem)
2068 {
2069     oid_value curAttributeSet = attributeSet;
2070     AttrType use;
2071     const char *use_string = 0;
2072
2073     attr_init(&use, zapt, 1);
2074     attr_find_ex(&use, &curAttributeSet, &use_string);
2075
2076     if (!use_string || *use_string != '/')
2077         return -1;
2078
2079     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2080 }
2081
2082
2083
2084 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2085                         int reg_type, const char *term, int use,
2086                         oid_value curAttributeSet, NMEM rset_nmem)
2087 {
2088     RSET rset;
2089     struct grep_info grep_info;
2090     char term_dict[2048];
2091     char ord_buf[32];
2092     int prefix_len = 0;
2093     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, curAttributeSet, use);
2094     int ord_len, i, r, max_pos;
2095     int term_type = Z_Term_characterString;
2096     const char *flags = "void";
2097
2098     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0', stream))
2099         return rsnull_create (rset_nmem,key_it_ctrl);
2100
2101     if (ord < 0)
2102         return rsnull_create (rset_nmem,key_it_ctrl);
2103     if (prefix_len)
2104         term_dict[prefix_len++] = '|';
2105     else
2106         term_dict[prefix_len++] = '(';
2107
2108     ord_len = key_SU_encode (ord, ord_buf);
2109     for (i = 0; i<ord_len; i++)
2110     {
2111         term_dict[prefix_len++] = 1;
2112         term_dict[prefix_len++] = ord_buf[i];
2113     }
2114     term_dict[prefix_len++] = ')';
2115     term_dict[prefix_len++] = 1;
2116     term_dict[prefix_len++] = reg_type;
2117
2118     strcpy(term_dict+prefix_len, term);
2119
2120     grep_info.isam_p_indx = 0;
2121     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2122                           &grep_info, &max_pos, 0, grep_handle);
2123     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2124              grep_info.isam_p_indx);
2125     rset = rset_trunc(zh, grep_info.isam_p_buf,
2126                       grep_info.isam_p_indx, term, strlen(term),
2127                       flags, 1, term_type,rset_nmem,
2128                       key_it_ctrl, key_it_ctrl->scope);
2129     grep_info_delete(&grep_info);
2130     return rset;
2131 }
2132
2133 static RSET rpn_search_xpath(ZebraHandle zh,
2134                              oid_value attributeSet,
2135                              int num_bases, char **basenames,
2136                              NMEM stream, const char *rank_type, RSET rset,
2137                              int xpath_len, struct xpath_location_step *xpath,
2138                              NMEM rset_nmem)
2139 {
2140     oid_value curAttributeSet = attributeSet;
2141     int base_no;
2142     int i;
2143
2144     if (xpath_len < 0)
2145         return rset;
2146
2147     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2148     for (i = 0; i<xpath_len; i++)
2149     {
2150         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2151
2152     }
2153
2154     curAttributeSet = VAL_IDXPATH;
2155
2156     /*
2157       //a    ->    a/.*
2158       //a/b  ->    b/a/.*
2159       /a     ->    a/
2160       /a/b   ->    b/a/
2161
2162       /      ->    none
2163
2164    a[@attr = value]/b[@other = othervalue]
2165
2166  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2167  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2168  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2169  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2170  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2171  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2172
2173     */
2174
2175     dict_grep_cmap (zh->reg->dict, 0, 0);
2176
2177     for (base_no = 0; base_no < num_bases; base_no++)
2178     {
2179         int level = xpath_len;
2180         int first_path = 1;
2181
2182         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2183         {
2184             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
2185             zh->errString = basenames[base_no];
2186             return rset;
2187         }
2188         while (--level >= 0)
2189         {
2190             char xpath_rev[128];
2191             int i, len;
2192             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2193
2194             *xpath_rev = 0;
2195             len = 0;
2196             for (i = level; i >= 1; --i)
2197             {
2198                 const char *cp = xpath[i].part;
2199                 if (*cp)
2200                 {
2201                     for (;*cp; cp++)
2202                         if (*cp == '*')
2203                         {
2204                             memcpy (xpath_rev + len, "[^/]*", 5);
2205                             len += 5;
2206                         }
2207                         else if (*cp == ' ')
2208                         {
2209
2210                             xpath_rev[len++] = 1;
2211                             xpath_rev[len++] = ' ';
2212                         }
2213
2214                         else
2215                             xpath_rev[len++] = *cp;
2216                     xpath_rev[len++] = '/';
2217                 }
2218                 else if (i == 1)  /* // case */
2219                 {
2220                     xpath_rev[len++] = '.';
2221                     xpath_rev[len++] = '*';
2222                 }
2223             }
2224             xpath_rev[len] = 0;
2225
2226             if (xpath[level].predicate &&
2227                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2228                 xpath[level].predicate->u.relation.name[0])
2229             {
2230                 WRBUF wbuf = wrbuf_alloc();
2231                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2232                 if (xpath[level].predicate->u.relation.value)
2233                 {
2234                     const char *cp = xpath[level].predicate->u.relation.value;
2235                     wrbuf_putc(wbuf, '=');
2236
2237                     while (*cp)
2238                     {
2239                         if (strchr(REGEX_CHARS, *cp))
2240                             wrbuf_putc(wbuf, '\\');
2241                         wrbuf_putc(wbuf, *cp);
2242                         cp++;
2243                     }
2244                 }
2245                 wrbuf_puts(wbuf, "");
2246                 rset_attr = xpath_trunc(
2247                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2248                     curAttributeSet,rset_nmem);
2249                 wrbuf_free(wbuf, 1);
2250             }
2251             else
2252             {
2253                 if (!first_path)
2254                     continue;
2255             }
2256             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2257             if (strlen(xpath_rev))
2258             {
2259                 rset_start_tag = xpath_trunc(zh, stream, '0',
2260                         xpath_rev, 1, curAttributeSet, rset_nmem);
2261
2262                 rset_end_tag = xpath_trunc(zh, stream, '0',
2263                         xpath_rev, 2, curAttributeSet, rset_nmem);
2264
2265                 rset = rsbetween_create(rset_nmem, key_it_ctrl,
2266                                         key_it_ctrl->scope,
2267                                         rset_start_tag, rset,
2268                                         rset_end_tag, rset_attr);
2269             }
2270             first_path = 0;
2271         }
2272     }
2273
2274     return rset;
2275 }
2276
2277
2278
2279 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2280                                 oid_value attributeSet, NMEM stream,
2281                                 Z_SortKeySpecList *sort_sequence,
2282                                 int num_bases, char **basenames,
2283                                 NMEM rset_nmem,
2284                                 RSET *rset)
2285 {
2286     ZEBRA_RES res = ZEBRA_OK;
2287     unsigned reg_id;
2288     char *search_type = NULL;
2289     char rank_type[128];
2290     int complete_flag;
2291     int sort_flag;
2292     char termz[IT_MAX_WORD+1];
2293     int xpath_len;
2294     int xpath_use = 0;
2295     struct xpath_location_step xpath[10];
2296
2297     if (!log_level_set)
2298     {
2299         log_level_rpn = yaz_log_module_level("rpn");
2300         log_level_set = 1;
2301     }
2302     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2303                     rank_type, &complete_flag, &sort_flag);
2304
2305     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2306     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2307     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2308     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2309
2310     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2311         return ZEBRA_FAIL;
2312
2313     if (sort_flag)
2314         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2315                              rank_type, rset);
2316     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2317     if (xpath_len >= 0)
2318     {
2319         xpath_use = 1016;
2320         if (xpath[xpath_len-1].part[0] == '@')
2321             xpath_use = 1015;
2322     }
2323
2324     if (!strcmp(search_type, "phrase"))
2325     {
2326         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2327                                     reg_id, complete_flag, rank_type,
2328                                     xpath_use,
2329                                     num_bases, basenames, rset_nmem,
2330                                     rset);
2331     }
2332     else if (!strcmp(search_type, "and-list"))
2333     {
2334         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2335                                       reg_id, complete_flag, rank_type,
2336                                       xpath_use,
2337                                       num_bases, basenames, rset_nmem,
2338                                       rset);
2339     }
2340     else if (!strcmp(search_type, "or-list"))
2341     {
2342         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2343                                      reg_id, complete_flag, rank_type,
2344                                      xpath_use,
2345                                      num_bases, basenames, rset_nmem,
2346                                      rset);
2347     }
2348     else if (!strcmp(search_type, "local"))
2349     {
2350         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2351                                    rank_type, rset_nmem, rset);
2352     }
2353     else if (!strcmp(search_type, "numeric"))
2354     {
2355         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2356                                      reg_id, complete_flag, rank_type,
2357                                      xpath_use,
2358                                      num_bases, basenames, rset_nmem, rset);
2359     }
2360     else
2361     {
2362         zh->errCode = YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE;
2363         return ZEBRA_FAIL;
2364     }
2365     if (res != ZEBRA_OK)
2366         return res;
2367     if (!*rset)
2368         return ZEBRA_FAIL;
2369     *rset = rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2370                              stream, rank_type, *rset,
2371                              xpath_len, xpath, rset_nmem);
2372     if (!*rset)
2373         return ZEBRA_FAIL;
2374     return ZEBRA_OK;
2375 }
2376
2377 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2378                                       oid_value attributeSet,
2379                                       NMEM stream, NMEM rset_nmem,
2380                                       Z_SortKeySpecList *sort_sequence,
2381                                       int num_bases, char **basenames,
2382                                       RSET **result_sets, int *num_result_sets,
2383                                       Z_Operator *parent_op);
2384
2385 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2386                          oid_value attributeSet,
2387                          NMEM stream, NMEM rset_nmem,
2388                          Z_SortKeySpecList *sort_sequence,
2389                          int num_bases, char **basenames,
2390                          RSET *result_set)
2391 {
2392     RSET *result_sets = 0;
2393     int num_result_sets = 0;
2394     ZEBRA_RES res = rpn_search_structure(zh, zs, attributeSet,
2395                                          stream, rset_nmem,
2396                                          sort_sequence,
2397                                          num_bases, basenames,
2398                                          &result_sets, &num_result_sets,
2399                                          0 /* no op */);
2400     if (res != ZEBRA_OK)
2401     {
2402         int i;
2403         for (i = 0; i<num_result_sets; i++)
2404             rset_delete(result_sets[i]);
2405         *result_set = 0;
2406         return res;
2407     }
2408     assert(num_result_sets == 1);
2409     assert(result_sets);
2410     assert(*result_sets);
2411     *result_set = *result_sets;
2412     return ZEBRA_OK;
2413 }
2414
2415 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2416                                oid_value attributeSet,
2417                                NMEM stream, NMEM rset_nmem,
2418                                Z_SortKeySpecList *sort_sequence,
2419                                int num_bases, char **basenames,
2420                                RSET **result_sets, int *num_result_sets,
2421                                Z_Operator *parent_op)
2422 {
2423     *num_result_sets = 0;
2424     if (zs->which == Z_RPNStructure_complex)
2425     {
2426         ZEBRA_RES res;
2427         Z_Operator *zop = zs->u.complex->roperator;
2428         RSET *result_sets_l = 0;
2429         int num_result_sets_l = 0;
2430         RSET *result_sets_r = 0;
2431         int num_result_sets_r = 0;
2432
2433         res = rpn_search_structure(zh, zs->u.complex->s1,
2434                                    attributeSet, stream, rset_nmem,
2435                                    sort_sequence,
2436                                    num_bases, basenames,
2437                                    &result_sets_l, &num_result_sets_l,
2438                                    zop);
2439         if (res != ZEBRA_OK)
2440         {
2441             int i;
2442             for (i = 0; i<num_result_sets_l; i++)
2443                 rset_delete(result_sets_l[i]);
2444             return res;
2445         }
2446         res = rpn_search_structure(zh, zs->u.complex->s2,
2447                                    attributeSet, stream, rset_nmem,
2448                                    sort_sequence,
2449                                    num_bases, basenames,
2450                                    &result_sets_r, &num_result_sets_r,
2451                                    zop);
2452         if (res != ZEBRA_OK)
2453         {
2454             int i;
2455             for (i = 0; i<num_result_sets_l; i++)
2456                 rset_delete(result_sets_l[i]);
2457             for (i = 0; i<num_result_sets_r; i++)
2458                 rset_delete(result_sets_r[i]);
2459             return res;
2460         }
2461
2462         /* make a new list of result for all children */
2463         *num_result_sets = num_result_sets_l + num_result_sets_r;
2464         *result_sets = nmem_malloc(stream, *num_result_sets *
2465                                    sizeof(**result_sets));
2466         memcpy(*result_sets, result_sets_l,
2467                num_result_sets_l * sizeof(**result_sets));
2468         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2469                num_result_sets_r * sizeof(**result_sets));
2470
2471         if (!parent_op || parent_op->which != zop->which
2472             || (zop->which != Z_Operator_and &&
2473                 zop->which != Z_Operator_or))
2474         {
2475             /* parent node different from this one (or non-present) */
2476             /* we must combine result sets now */
2477             RSET rset;
2478             switch (zop->which)
2479             {
2480             case Z_Operator_and:
2481                 rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2482                                           key_it_ctrl->scope,
2483                                           *num_result_sets, *result_sets);
2484                 break;
2485             case Z_Operator_or:
2486                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2487                                          key_it_ctrl->scope,
2488                                          *num_result_sets, *result_sets);
2489                 break;
2490             case Z_Operator_and_not:
2491                 rset = rsbool_create_not(rset_nmem, key_it_ctrl,
2492                                          key_it_ctrl->scope,
2493                                          (*result_sets)[0],
2494                                          (*result_sets)[1]);
2495                 break;
2496             case Z_Operator_prox:
2497                 if (zop->u.prox->which != Z_ProximityOperator_known)
2498                 {
2499                     zh->errCode = YAZ_BIB1_UNSUPP_PROX_UNIT_CODE;
2500                     return ZEBRA_FAIL;
2501                 }
2502                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2503                 {
2504                     char *val = (char *) nmem_malloc(stream, 16);
2505                     zh->errCode = YAZ_BIB1_UNSUPP_PROX_UNIT_CODE;
2506                     zh->errString = val;
2507                     sprintf(val, "%d", *zop->u.prox->u.known);
2508                     return ZEBRA_FAIL;
2509                 }
2510                 else
2511                 {
2512                     rset = rsprox_create(rset_nmem, key_it_ctrl,
2513                                          key_it_ctrl->scope,
2514                                          *num_result_sets, *result_sets,
2515                                          *zop->u.prox->ordered,
2516                                          (!zop->u.prox->exclusion ?
2517                                           0 : *zop->u.prox->exclusion),
2518                                          *zop->u.prox->relationType,
2519                                          *zop->u.prox->distance );
2520                 }
2521                 break;
2522             default:
2523                 zh->errCode = YAZ_BIB1_OPERATOR_UNSUPP;
2524                 return ZEBRA_FAIL;
2525             }
2526             *num_result_sets = 1;
2527             *result_sets = nmem_malloc(stream, *num_result_sets *
2528                                        sizeof(**result_sets));
2529             (*result_sets)[0] = rset;
2530         }
2531     }
2532     else if (zs->which == Z_RPNStructure_simple)
2533     {
2534         RSET rset;
2535         ZEBRA_RES res;
2536
2537         if (zs->u.simple->which == Z_Operand_APT)
2538         {
2539             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2540             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2541                                  attributeSet, stream, sort_sequence,
2542                                  num_bases, basenames, rset_nmem, &rset);
2543             if (res != ZEBRA_OK)
2544                 return res;
2545         }
2546         else if (zs->u.simple->which == Z_Operand_resultSetId)
2547         {
2548             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2549             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2550             if (!rset)
2551             {
2552                 zh->errCode = YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST;
2553                 zh->errString =
2554                     nmem_strdup(stream, zs->u.simple->u.resultSetId);
2555                 return ZEBRA_FAIL;
2556             }
2557             rset_dup(rset);
2558         }
2559         else
2560         {
2561             zh->errCode = YAZ_BIB1_UNSUPP_SEARCH;
2562             return ZEBRA_FAIL;
2563         }
2564         *num_result_sets = 1;
2565         *result_sets = nmem_malloc(stream, *num_result_sets *
2566                                    sizeof(**result_sets));
2567         (*result_sets)[0] = rset;
2568     }
2569     else
2570     {
2571         zh->errCode = YAZ_BIB1_UNSUPP_SEARCH;
2572         return ZEBRA_FAIL;
2573     }
2574     return ZEBRA_OK;
2575 }
2576
2577 struct scan_info_entry {
2578     char *term;
2579     ISAM_P isam_p;
2580 };
2581
2582 struct scan_info {
2583     struct scan_info_entry *list;
2584     ODR odr;
2585     int before, after;
2586     char prefix[20];
2587 };
2588
2589 static int scan_handle (char *name, const char *info, int pos, void *client)
2590 {
2591     int len_prefix, idx;
2592     struct scan_info *scan_info = (struct scan_info *) client;
2593
2594     len_prefix = strlen(scan_info->prefix);
2595     if (memcmp (name, scan_info->prefix, len_prefix))
2596         return 1;
2597     if (pos > 0)
2598         idx = scan_info->after - pos + scan_info->before;
2599     else
2600         idx = - pos - 1;
2601     if (idx < 0)
2602         return 0;
2603     scan_info->list[idx].term = (char *)
2604         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2605     strcpy(scan_info->list[idx].term, name + len_prefix);
2606     assert (*info == sizeof(ISAM_P));
2607     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2608     return 0;
2609 }
2610
2611 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2612                                char **dst, const char *src)
2613 {
2614     char term_src[IT_MAX_WORD];
2615     char term_dst[IT_MAX_WORD];
2616
2617     term_untrans (zh, reg_type, term_src, src);
2618
2619     if (zh->iconv_from_utf8 != 0)
2620     {
2621         int len;
2622         char *inbuf = term_src;
2623         size_t inleft = strlen(term_src);
2624         char *outbuf = term_dst;
2625         size_t outleft = sizeof(term_dst)-1;
2626         size_t ret;
2627
2628         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2629                          &outbuf, &outleft);
2630         if (ret == (size_t)(-1))
2631             len = 0;
2632         else
2633             len = outbuf - term_dst;
2634         *dst = nmem_malloc(stream, len + 1);
2635         if (len > 0)
2636             memcpy (*dst, term_dst, len);
2637         (*dst)[len] = '\0';
2638     }
2639     else
2640         *dst = nmem_strdup(stream, term_src);
2641 }
2642
2643 static void count_set (RSET r, int *count)
2644 {
2645     zint psysno = 0;
2646     int kno = 0;
2647     struct it_key key;
2648     RSFD rfd;
2649
2650     yaz_log(YLOG_DEBUG, "count_set");
2651
2652     *count = 0;
2653     rfd = rset_open (r, RSETF_READ);
2654     while (rset_read (rfd, &key,0 /* never mind terms */))
2655     {
2656         if (key.mem[0] != psysno)
2657         {
2658             psysno = key.mem[0];
2659             (*count)++;
2660         }
2661         kno++;
2662     }
2663     rset_close (rfd);
2664     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2665 }
2666
2667 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2668                    oid_value attributeset,
2669                    int num_bases, char **basenames,
2670                    int *position, int *num_entries, ZebraScanEntry **list,
2671                    int *is_partial, RSET limit_set, int return_zero)
2672 {
2673     int i;
2674     int pos = *position;
2675     int num = *num_entries;
2676     int before;
2677     int after;
2678     int base_no;
2679     char termz[IT_MAX_WORD+20];
2680     AttrType use;
2681     int use_value;
2682     const char *use_string = 0;
2683     struct scan_info *scan_info_array;
2684     ZebraScanEntry *glist;
2685     int ords[32], ord_no = 0;
2686     int ptr[32];
2687
2688     int bases_ok = 0;     /* no of databases with OK attribute */
2689     int errCode = 0;      /* err code (if any is not OK) */
2690     char *errString = 0;  /* addinfo */
2691
2692     unsigned reg_id;
2693     char *search_type = NULL;
2694     char rank_type[128];
2695     int complete_flag;
2696     int sort_flag;
2697     NMEM rset_nmem = NULL;
2698
2699     *list = 0;
2700     *is_partial = 0;
2701
2702     if (attributeset == VAL_NONE)
2703         attributeset = VAL_BIB1;
2704
2705     if (!limit_set)
2706     {
2707         AttrType termset;
2708         int termset_value_numeric;
2709         const char *termset_value_string;
2710         attr_init(&termset, zapt, 8);
2711         termset_value_numeric =
2712             attr_find_ex(&termset, NULL, &termset_value_string);
2713         if (termset_value_numeric != -1)
2714         {
2715             char resname[32];
2716             const char *termset_name = 0;
2717
2718             if (termset_value_numeric != -2)
2719             {
2720
2721                 sprintf(resname, "%d", termset_value_numeric);
2722                 termset_name = resname;
2723             }
2724             else
2725                 termset_name = termset_value_string;
2726
2727             limit_set = resultSetRef (zh, termset_name);
2728         }
2729     }
2730
2731     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2732             pos, num, attributeset);
2733
2734     attr_init(&use, zapt, 1);
2735     use_value = attr_find_ex(&use, &attributeset, &use_string);
2736
2737     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2738                         rank_type, &complete_flag, &sort_flag))
2739     {
2740         *num_entries = 0;
2741         zh->errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE;
2742         return ZEBRA_FAIL;
2743     }
2744     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2745
2746     if (use_value == -1)
2747         use_value = 1016;
2748     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2749     {
2750         data1_local_attribute *local_attr;
2751         attent attp;
2752         int ord;
2753
2754         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2755         {
2756             zh->errString = basenames[base_no];
2757             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
2758             *num_entries = 0;
2759             return ZEBRA_FAIL;
2760         }
2761
2762         if (use_string &&
2763             (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2764                                                 use_string)) >= 0)
2765         {
2766             /* we have a match for a raw string attribute */
2767             if (ord > 0)
2768                 ords[ord_no++] = ord;
2769             attp.local_attributes = 0;  /* no more attributes */
2770         }
2771         else
2772         {
2773             int r;
2774
2775             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2776                                       use_string)))
2777             {
2778                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2779                         attributeset, use_value);
2780                 if (r == -1)
2781                 {
2782                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2783                     if (use_string)
2784                         errString = odr_strdup(stream, use_string);
2785                     else
2786                     {
2787                         char val_str[32];
2788                         sprintf(val_str, "%d", use_value);
2789                         errString = odr_strdup(stream, val_str);
2790                     }
2791                 }
2792                 else
2793                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
2794                 continue;
2795             }
2796         }
2797         bases_ok++;
2798         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2799              local_attr = local_attr->next)
2800         {
2801             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
2802                                               attp.attset_ordinal,
2803                                               local_attr->local);
2804             if (ord > 0)
2805                 ords[ord_no++] = ord;
2806         }
2807     }
2808     if (!bases_ok && errCode)
2809     {
2810         zh->errCode = errCode;
2811         zh->errString = errString;
2812         *num_entries = 0;
2813         return ZEBRA_FAIL;
2814     }
2815     if (ord_no == 0)
2816     {
2817         *num_entries = 0;
2818         return ZEBRA_OK;
2819     }
2820     /* prepare dictionary scanning */
2821     if (pos <= 0)
2822     {
2823         zh->errCode = YAZ_BIB1_SCAN_UNSUPP_VALUE_OF_POSITION_IN_RESPONSE;
2824         *num_entries = 0;
2825         return ZEBRA_FAIL;
2826     }
2827     if (num < 1)
2828     {
2829         *num_entries = 0;
2830         return ZEBRA_OK;
2831     }
2832     before = pos-1;
2833     after = 1+num-pos;
2834     yaz_log(YLOG_EBUG, "rpn_scan pos=%d num=%d before=%d "
2835             "after=%d before+after=%d",
2836             pos, num, before, after, before+after);
2837     scan_info_array = (struct scan_info *)
2838         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2839     for (i = 0; i < ord_no; i++)
2840     {
2841         int j, prefix_len = 0;
2842         int before_tmp = before, after_tmp = after;
2843         struct scan_info *scan_info = scan_info_array + i;
2844         struct rpn_char_map_info rcmi;
2845
2846         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2847
2848         scan_info->before = before;
2849         scan_info->after = after;
2850         scan_info->odr = stream;
2851
2852         scan_info->list = (struct scan_info_entry *)
2853             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2854         for (j = 0; j<before+after; j++)
2855             scan_info->list[j].term = NULL;
2856
2857         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2858         termz[prefix_len++] = reg_id;
2859         termz[prefix_len] = 0;
2860         strcpy(scan_info->prefix, termz);
2861
2862         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2863             return ZEBRA_FAIL;
2864
2865         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2866                   scan_info, scan_handle);
2867     }
2868     glist = (ZebraScanEntry *)
2869         odr_malloc(stream, (before+after)*sizeof(*glist));
2870
2871     rset_nmem = nmem_create();
2872
2873     /* consider terms after main term */
2874     for (i = 0; i < ord_no; i++)
2875         ptr[i] = before;
2876
2877     *is_partial = 0;
2878     for (i = 0; i<after; i++)
2879     {
2880         int j, j0 = -1;
2881         const char *mterm = NULL;
2882         const char *tst;
2883         RSET rset;
2884
2885         for (j = 0; j < ord_no; j++)
2886         {
2887             if (ptr[j] < before+after && ptr[j] >= 0 &&
2888                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2889                 (!mterm || strcmp (tst, mterm) < 0))
2890             {
2891                 j0 = j;
2892                 mterm = tst;
2893             }
2894         }
2895         if (j0 == -1)
2896             break;
2897         scan_term_untrans(zh, stream->mem, reg_id,
2898                           &glist[i+before].term, mterm);
2899         rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2900                           glist[i+before].term, strlen(glist[i+before].term),
2901                           NULL, 0, zapt->term->which, rset_nmem,
2902                           key_it_ctrl,key_it_ctrl->scope);
2903         ptr[j0]++;
2904         for (j = j0+1; j<ord_no; j++)
2905         {
2906             if (ptr[j] < before+after && ptr[j] >= 0 &&
2907                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2908                 !strcmp (tst, mterm))
2909             {
2910                 RSET rsets[2];
2911
2912                 rsets[0] = rset;
2913                 rsets[1] =
2914                     rset_trunc(zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2915                                glist[i+before].term,
2916                                strlen(glist[i+before].term), NULL, 0,
2917                                zapt->term->which,rset_nmem,
2918                                key_it_ctrl, key_it_ctrl->scope);
2919                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2920                                          2, key_it_ctrl->scope, rsets);
2921                 ptr[j]++;
2922             }
2923         }
2924         if (limit_set)
2925         {
2926             RSET rsets[2];
2927             rsets[0] = rset;
2928             rsets[1] = rset_dup(limit_set);
2929
2930             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2931                                       key_it_ctrl->scope, 2, rsets);
2932         }
2933         count_set(rset, &glist[i+before].occurrences);
2934         rset_delete(rset);
2935     }
2936     if (i < after)
2937     {
2938         *num_entries -= (after-i);
2939         *is_partial = 1;
2940     }
2941     /* consider terms before main term */
2942     for (i = 0; i<ord_no; i++)
2943         ptr[i] = 0;
2944
2945     for (i = 0; i<before; i++)
2946     {
2947         int j, j0 = -1;
2948         const char *mterm = NULL;
2949         const char *tst;
2950         RSET rset;
2951
2952         for (j = 0; j <ord_no; j++)
2953         {
2954             if (ptr[j] < before && ptr[j] >= 0 &&
2955                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2956                 (!mterm || strcmp (tst, mterm) > 0))
2957             {
2958                 j0 = j;
2959                     mterm = tst;
2960             }
2961         }
2962         if (j0 == -1)
2963             break;
2964
2965         scan_term_untrans (zh, stream->mem, reg_id,
2966                            &glist[before-1-i].term, mterm);
2967
2968         rset = rset_trunc
2969             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2970              glist[before-1-i].term, strlen(glist[before-1-i].term),
2971              NULL, 0, zapt->term->which,rset_nmem,
2972              key_it_ctrl,key_it_ctrl->scope);
2973
2974         ptr[j0]++;
2975
2976         for (j = j0+1; j<ord_no; j++)
2977         {
2978             if (ptr[j] < before && ptr[j] >= 0 &&
2979                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2980                 !strcmp (tst, mterm))
2981             {
2982                 RSET rsets[2];
2983
2984                 rsets[0] = rset;
2985                 rsets[1] = rset_trunc(
2986                     zh,
2987                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2988                     glist[before-1-i].term,
2989                     strlen(glist[before-1-i].term), NULL, 0,
2990                     zapt->term->which, rset_nmem,
2991                     key_it_ctrl, key_it_ctrl->scope);
2992                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2993                                          2, key_it_ctrl->scope, rsets);
2994
2995                 ptr[j]++;
2996             }
2997         }
2998         if (limit_set)
2999         {
3000             RSET rsets[2];
3001             rsets[0] = rset;
3002             rsets[1] = rset_dup(limit_set);
3003
3004             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
3005                                       key_it_ctrl->scope, 2, rsets);
3006         }
3007         count_set (rset, &glist[before-1-i].occurrences);
3008         rset_delete (rset);
3009     }
3010     i = before-i;
3011     if (i)
3012     {
3013         *is_partial = 1;
3014         *position -= i;
3015         *num_entries -= i;
3016     }
3017
3018     nmem_destroy(rset_nmem);
3019     *list = glist + i;               /* list is set to first 'real' entry */
3020
3021     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3022             *position, *num_entries);
3023     return ZEBRA_OK;
3024 }
3025