index/zrpn.c

   1 /* $Id: zrpn.c,v 1.206 2005-11-02 11:43:26 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 typedef struct
  47 {
  48     int type;
  49     int major;
  50     int minor;
  51     Z_AttributesPlusTerm *zapt;
  52 } AttrType;
  53
  54 static int log_level_set = 0;
  55 static int log_level_rpn = 0;
  56
  57 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  58 {
  59     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  60     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  61 #if 0
  62     if (out && *out)
  63     {
  64         const char *outp = *out;
  65         yaz_log(YLOG_LOG, "---");
  66         while (*outp)
  67         {
  68             yaz_log(YLOG_LOG, "%02X", *outp);
  69             outp++;
  70         }
  71     }
  72 #endif
  73     return out;
  74 }
  75
  76 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  77                                   struct rpn_char_map_info *map_info)
  78 {
  79     map_info->zm = reg->zebra_maps;
  80     map_info->reg_type = reg_type;
  81     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  82 }
  83
  84 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  85                         const char **string_value)
  86 {
  87     int num_attributes;
  88
  89     num_attributes = src->zapt->attributes->num_attributes;
  90     while (src->major < num_attributes)
  91     {
  92         Z_AttributeElement *element;
  93
  94         element = src->zapt->attributes->attributes[src->major];
  95         if (src->type == *element->attributeType)
  96         {
  97             switch (element->which)
  98             {
  99             case Z_AttributeValue_numeric:
 100                 ++(src->major);
 101                 if (element->attributeSet && attributeSetP)
 102                 {
 103                     oident *attrset;
 104
 105                     attrset = oid_getentbyoid(element->attributeSet);
 106                     *attributeSetP = attrset->value;
 107                 }
 108                 return *element->value.numeric;
 109                 break;
 110             case Z_AttributeValue_complex:
 111                 if (src->minor >= element->value.complex->num_list)
 112                     break;
 113                 if (element->attributeSet && attributeSetP)
 114                 {
 115                     oident *attrset;
 116
 117                     attrset = oid_getentbyoid(element->attributeSet);
 118                     *attributeSetP = attrset->value;
 119                 }
 120                 if (element->value.complex->list[src->minor]->which ==
 121                     Z_StringOrNumeric_numeric)
 122                 {
 123                     ++(src->minor);
 124                     return
 125                         *element->value.complex->list[src->minor-1]->u.numeric;
 126                 }
 127                 else if (element->value.complex->list[src->minor]->which ==
 128                          Z_StringOrNumeric_string)
 129                 {
 130                     if (!string_value)
 131                         break;
 132                     ++(src->minor);
 133                     *string_value =
 134                         element->value.complex->list[src->minor-1]->u.string;
 135                     return -2;
 136                 }
 137                 else
 138                     break;
 139             default:
 140                 assert(0);
 141             }
 142         }
 143         ++(src->major);
 144     }
 145     return -1;
 146 }
 147
 148 static int attr_find(AttrType *src, oid_value *attributeSetP)
 149 {
 150     return attr_find_ex(src, attributeSetP, 0);
 151 }
 152
 153 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 154                        int type)
 155 {
 156     src->zapt = zapt;
 157     src->type = type;
 158     src->major = 0;
 159     src->minor = 0;
 160 }
 161
 162 #define TERM_COUNT
 163
 164 struct grep_info {
 165 #ifdef TERM_COUNT
 166     int *term_no;
 167 #endif
 168     ISAM_P *isam_p_buf;
 169     int isam_p_size;
 170     int isam_p_indx;
 171     ZebraHandle zh;
 172     int reg_type;
 173     ZebraSet termset;
 174 };
 175
 176 void zebra_term_untrans(ZebraHandle zh, int reg_type,
 177                         char *dst, const char *src)
 178 {
 179     int len = 0;
 180     while (*src)
 181     {
 182         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 183                                            reg_type, &src);
 184         if (!cp && len < IT_MAX_WORD-1)
 185             dst[len++] = *src++;
 186         else
 187             while (*cp && len < IT_MAX_WORD-1)
 188                 dst[len++] = *cp++;
 189     }
 190     dst[len] = '\0';
 191 }
 192
 193 static void add_isam_p(const char *name, const char *info,
 194                        struct grep_info *p)
 195 {
 196     if (!log_level_set)
 197     {
 198         log_level_rpn = yaz_log_module_level("rpn");
 199         log_level_set = 1;
 200     }
 201     if (p->isam_p_indx == p->isam_p_size)
 202     {
 203         ISAM_P *new_isam_p_buf;
 204 #ifdef TERM_COUNT
 205         int *new_term_no;
 206 #endif
 207         p->isam_p_size = 2*p->isam_p_size + 100;
 208         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 209                                             p->isam_p_size);
 210         if (p->isam_p_buf)
 211         {
 212             memcpy(new_isam_p_buf, p->isam_p_buf,
 213                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 214             xfree(p->isam_p_buf);
 215         }
 216         p->isam_p_buf = new_isam_p_buf;
 217
 218 #ifdef TERM_COUNT
 219         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 220         if (p->term_no)
 221         {
 222             memcpy(new_term_no, p->isam_p_buf,
 223                     p->isam_p_indx * sizeof(*p->term_no));
 224             xfree(p->term_no);
 225         }
 226         p->term_no = new_term_no;
 227 #endif
 228     }
 229     assert(*info == sizeof(*p->isam_p_buf));
 230     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 231
 232 #if 1
 233     if (p->termset)
 234     {
 235         const char *db;
 236         int set, use;
 237         char term_tmp[IT_MAX_WORD];
 238         int ord = 0;
 239         int len = key_SU_decode (&ord, (const unsigned char *) name);
 240
 241         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 242         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 243         zebraExplain_lookup_ord (p->zh->reg->zei,
 244                                  ord, 0 /* index_type */, &db, &set, &use);
 245         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 246
 247         resultSetAddTerm(p->zh, p->termset, name[len], db,
 248                          set, use, term_tmp);
 249     }
 250 #endif
 251     (p->isam_p_indx)++;
 252 }
 253
 254 static int grep_handle(char *name, const char *info, void *p)
 255 {
 256     add_isam_p(name, info, (struct grep_info *) p);
 257     return 0;
 258 }
 259
 260 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 261                     const char *ct1, const char *ct2, int first)
 262 {
 263     const char *s1, *s0 = *src;
 264     const char **map;
 265
 266     /* skip white space */
 267     while (*s0)
 268     {
 269         if (ct1 && strchr(ct1, *s0))
 270             break;
 271         if (ct2 && strchr(ct2, *s0))
 272             break;
 273         s1 = s0;
 274         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 275         if (**map != *CHR_SPACE)
 276             break;
 277         s0 = s1;
 278     }
 279     *src = s0;
 280     return *s0;
 281 }
 282
 283
 284 static void esc_str(char *out_buf, int out_size,
 285                     const char *in_buf, int in_size)
 286 {
 287     int k;
 288
 289     assert(out_buf);
 290     assert(in_buf);
 291     assert(out_size > 20);
 292     *out_buf = '\0';
 293     for (k = 0; k<in_size; k++)
 294     {
 295         int c = in_buf[k] & 0xff;
 296         int pc;
 297         if (c < 32 || c > 126)
 298             pc = '?';
 299         else
 300             pc = c;
 301         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 302         if (strlen(out_buf) > out_size-20)
 303         {
 304             strcat(out_buf, "..");
 305             break;
 306         }
 307     }
 308 }
 309
 310 #define REGEX_CHARS " []()|.*+?!"
 311
 312 /* term_100: handle term, where trunc = none(no operators at all) */
 313 static int term_100(ZebraMaps zebra_maps, int reg_type,
 314                     const char **src, char *dst, int space_split,
 315                     char *dst_term)
 316 {
 317     const char *s0;
 318     const char **map;
 319     int i = 0;
 320     int j = 0;
 321
 322     const char *space_start = 0;
 323     const char *space_end = 0;
 324
 325     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 326         return 0;
 327     s0 = *src;
 328     while (*s0)
 329     {
 330         const char *s1 = s0;
 331         int q_map_match = 0;
 332         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 333                                 &q_map_match);
 334         if (space_split)
 335         {
 336             if (**map == *CHR_SPACE)
 337                 break;
 338         }
 339         else  /* complete subfield only. */
 340         {
 341             if (**map == *CHR_SPACE)
 342             {   /* save space mapping for later  .. */
 343                 space_start = s1;
 344                 space_end = s0;
 345                 continue;
 346             }
 347             else if (space_start)
 348             {   /* reload last space */
 349                 while (space_start < space_end)
 350                 {
 351                     if (strchr(REGEX_CHARS, *space_start))
 352                         dst[i++] = '\\';
 353                     dst_term[j++] = *space_start;
 354                     dst[i++] = *space_start++;
 355                 }
 356                 /* and reset */
 357                 space_start = space_end = 0;
 358             }
 359         }
 360         /* add non-space char */
 361         memcpy(dst_term+j, s1, s0 - s1);
 362         j += (s0 - s1);
 363         if (!q_map_match)
 364         {
 365             while (s1 < s0)
 366             {
 367                 if (strchr(REGEX_CHARS, *s1))
 368                     dst[i++] = '\\';
 369                 dst[i++] = *s1++;
 370             }
 371         }
 372         else
 373         {
 374             char tmpbuf[80];
 375             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 376
 377             strcpy(dst + i, map[0]);
 378             i += strlen(map[0]);
 379         }
 380     }
 381     dst[i] = '\0';
 382     dst_term[j] = '\0';
 383     *src = s0;
 384     return i;
 385 }
 386
 387 /* term_101: handle term, where trunc = Process # */
 388 static int term_101(ZebraMaps zebra_maps, int reg_type,
 389                     const char **src, char *dst, int space_split,
 390                     char *dst_term)
 391 {
 392     const char *s0;
 393     const char **map;
 394     int i = 0;
 395     int j = 0;
 396
 397     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 398         return 0;
 399     s0 = *src;
 400     while (*s0)
 401     {
 402         if (*s0 == '#')
 403         {
 404             dst[i++] = '.';
 405             dst[i++] = '*';
 406             dst_term[j++] = *s0++;
 407         }
 408         else
 409         {
 410             const char *s1 = s0;
 411             int q_map_match = 0;
 412             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 413                                     &q_map_match);
 414             if (space_split && **map == *CHR_SPACE)
 415                 break;
 416
 417             /* add non-space char */
 418             memcpy(dst_term+j, s1, s0 - s1);
 419             j += (s0 - s1);
 420             if (!q_map_match)
 421             {
 422                 while (s1 < s0)
 423                 {
 424                     if (strchr(REGEX_CHARS, *s1))
 425                         dst[i++] = '\\';
 426                     dst[i++] = *s1++;
 427                 }
 428             }
 429             else
 430             {
 431                 char tmpbuf[80];
 432                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 433
 434                 strcpy(dst + i, map[0]);
 435                 i += strlen(map[0]);
 436             }
 437         }
 438     }
 439     dst[i] = '\0';
 440     dst_term[j++] = '\0';
 441     *src = s0;
 442     return i;
 443 }
 444
 445 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 446 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 447                     char *dst, int *errors, int space_split,
 448                     char *dst_term)
 449 {
 450     int i = 0;
 451     int j = 0;
 452     const char *s0;
 453     const char **map;
 454
 455     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 456         return 0;
 457     s0 = *src;
 458     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 459         isdigit(((const unsigned char *)s0)[1]))
 460     {
 461         *errors = s0[1] - '0';
 462         s0 += 3;
 463         if (*errors > 3)
 464             *errors = 3;
 465     }
 466     while (*s0)
 467     {
 468         if (strchr("^\\()[].*+?|-", *s0))
 469         {
 470             dst_term[j++] = *s0;
 471             dst[i++] = *s0++;
 472         }
 473         else
 474         {
 475             const char *s1 = s0;
 476             int q_map_match = 0;
 477             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 478                                     &q_map_match);
 479             if (space_split && **map == *CHR_SPACE)
 480                 break;
 481
 482             /* add non-space char */
 483             memcpy(dst_term+j, s1, s0 - s1);
 484             j += (s0 - s1);
 485             if (!q_map_match)
 486             {
 487                 while (s1 < s0)
 488                 {
 489                     if (strchr(REGEX_CHARS, *s1))
 490                         dst[i++] = '\\';
 491                     dst[i++] = *s1++;
 492                 }
 493             }
 494             else
 495             {
 496                 char tmpbuf[80];
 497                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 498
 499                 strcpy(dst + i, map[0]);
 500                 i += strlen(map[0]);
 501             }
 502         }
 503     }
 504     dst[i] = '\0';
 505     dst_term[j] = '\0';
 506     *src = s0;
 507
 508     return i;
 509 }
 510
 511 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 512 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 513                     char *dst, int space_split, char *dst_term)
 514 {
 515     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 516                     dst_term);
 517 }
 518
 519
 520 /* term_104: handle term, where trunc = Process # and ! */
 521 static int term_104(ZebraMaps zebra_maps, int reg_type,
 522                     const char **src, char *dst, int space_split,
 523                     char *dst_term)
 524 {
 525     const char *s0;
 526     const char **map;
 527     int i = 0;
 528     int j = 0;
 529
 530     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 531         return 0;
 532     s0 = *src;
 533     while (*s0)
 534     {
 535         if (*s0 == '?')
 536         {
 537             dst_term[j++] = *s0++;
 538             if (*s0 >= '0' && *s0 <= '9')
 539             {
 540                 int limit = 0;
 541                 while (*s0 >= '0' && *s0 <= '9')
 542                 {
 543                     limit = limit * 10 + (*s0 - '0');
 544                     dst_term[j++] = *s0++;
 545                 }
 546                 if (limit > 20)
 547                     limit = 20;
 548                 while (--limit >= 0)
 549                 {
 550                     dst[i++] = '.';
 551                     dst[i++] = '?';
 552                 }
 553             }
 554             else
 555             {
 556                 dst[i++] = '.';
 557                 dst[i++] = '*';
 558             }
 559         }
 560         else if (*s0 == '*')
 561         {
 562             dst[i++] = '.';
 563             dst[i++] = '*';
 564             dst_term[j++] = *s0++;
 565         }
 566         else if (*s0 == '#')
 567         {
 568             dst[i++] = '.';
 569             dst_term[j++] = *s0++;
 570         }
 571         else
 572         {
 573             const char *s1 = s0;
 574             int q_map_match = 0;
 575             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 576                                     &q_map_match);
 577             if (space_split && **map == *CHR_SPACE)
 578                 break;
 579
 580             /* add non-space char */
 581             memcpy(dst_term+j, s1, s0 - s1);
 582             j += (s0 - s1);
 583             if (!q_map_match)
 584             {
 585                 while (s1 < s0)
 586                 {
 587                     if (strchr(REGEX_CHARS, *s1))
 588                         dst[i++] = '\\';
 589                     dst[i++] = *s1++;
 590                 }
 591             }
 592             else
 593             {
 594                 char tmpbuf[80];
 595                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 596
 597                 strcpy(dst + i, map[0]);
 598                 i += strlen(map[0]);
 599             }
 600         }
 601     }
 602     dst[i] = '\0';
 603     dst_term[j++] = '\0';
 604     *src = s0;
 605     return i;
 606 }
 607
 608 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 609 static int term_105(ZebraMaps zebra_maps, int reg_type,
 610                     const char **src, char *dst, int space_split,
 611                     char *dst_term, int right_truncate)
 612 {
 613     const char *s0;
 614     const char **map;
 615     int i = 0;
 616     int j = 0;
 617
 618     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 619         return 0;
 620     s0 = *src;
 621     while (*s0)
 622     {
 623         if (*s0 == '*')
 624         {
 625             dst[i++] = '.';
 626             dst[i++] = '*';
 627             dst_term[j++] = *s0++;
 628         }
 629         else if (*s0 == '!')
 630         {
 631             dst[i++] = '.';
 632             dst_term[j++] = *s0++;
 633         }
 634         else
 635         {
 636             const char *s1 = s0;
 637             int q_map_match = 0;
 638             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 639                                     &q_map_match);
 640             if (space_split && **map == *CHR_SPACE)
 641                 break;
 642
 643             /* add non-space char */
 644             memcpy(dst_term+j, s1, s0 - s1);
 645             j += (s0 - s1);
 646             if (!q_map_match)
 647             {
 648                 while (s1 < s0)
 649                 {
 650                     if (strchr(REGEX_CHARS, *s1))
 651                         dst[i++] = '\\';
 652                     dst[i++] = *s1++;
 653                 }
 654             }
 655             else
 656             {
 657                 char tmpbuf[80];
 658                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 659
 660                 strcpy(dst + i, map[0]);
 661                 i += strlen(map[0]);
 662             }
 663         }
 664     }
 665     if (right_truncate)
 666     {
 667         dst[i++] = '.';
 668         dst[i++] = '*';
 669     }
 670     dst[i] = '\0';
 671
 672     dst_term[j++] = '\0';
 673     *src = s0;
 674     return i;
 675 }
 676
 677
 678 /* gen_regular_rel - generate regular expression from relation
 679  *  val:     border value (inclusive)
 680  *  islt:    1 if <=; 0 if >=.
 681  */
 682 static void gen_regular_rel(char *dst, int val, int islt)
 683 {
 684     int dst_p;
 685     int w, d, i;
 686     int pos = 0;
 687     char numstr[20];
 688
 689     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 690     if (val >= 0)
 691     {
 692         if (islt)
 693             strcpy(dst, "(-[0-9]+|(");
 694         else
 695             strcpy(dst, "((");
 696     }
 697     else
 698     {
 699         if (!islt)
 700         {
 701             strcpy(dst, "([0-9]+|-(");
 702             dst_p = strlen(dst);
 703             islt = 1;
 704         }
 705         else
 706         {
 707             strcpy(dst, "(-(");
 708             islt = 0;
 709         }
 710         val = -val;
 711     }
 712     dst_p = strlen(dst);
 713     sprintf(numstr, "%d", val);
 714     for (w = strlen(numstr); --w >= 0; pos++)
 715     {
 716         d = numstr[w];
 717         if (pos > 0)
 718         {
 719             if (islt)
 720             {
 721                 if (d == '0')
 722                     continue;
 723                 d--;
 724             }
 725             else
 726             {
 727                 if (d == '9')
 728                     continue;
 729                 d++;
 730             }
 731         }
 732
 733         strcpy(dst + dst_p, numstr);
 734         dst_p = strlen(dst) - pos - 1;
 735
 736         if (islt)
 737         {
 738             if (d != '0')
 739             {
 740                 dst[dst_p++] = '[';
 741                 dst[dst_p++] = '0';
 742                 dst[dst_p++] = '-';
 743                 dst[dst_p++] = d;
 744                 dst[dst_p++] = ']';
 745             }
 746             else
 747                 dst[dst_p++] = d;
 748         }
 749         else
 750         {
 751             if (d != '9')
 752             {
 753                 dst[dst_p++] = '[';
 754                 dst[dst_p++] = d;
 755                 dst[dst_p++] = '-';
 756                 dst[dst_p++] = '9';
 757                 dst[dst_p++] = ']';
 758             }
 759             else
 760                 dst[dst_p++] = d;
 761         }
 762         for (i = 0; i<pos; i++)
 763         {
 764             dst[dst_p++] = '[';
 765             dst[dst_p++] = '0';
 766             dst[dst_p++] = '-';
 767             dst[dst_p++] = '9';
 768             dst[dst_p++] = ']';
 769         }
 770         dst[dst_p++] = '|';
 771     }
 772     dst[dst_p] = '\0';
 773     if (islt)
 774     {
 775         /* match everything less than 10^(pos-1) */
 776         strcat(dst, "0*");
 777         for (i = 1; i<pos; i++)
 778             strcat(dst, "[0-9]?");
 779     }
 780     else
 781     {
 782         /* match everything greater than 10^pos */
 783         for (i = 0; i <= pos; i++)
 784             strcat(dst, "[0-9]");
 785         strcat(dst, "[0-9]*");
 786     }
 787     strcat(dst, "))");
 788 }
 789
 790 void string_rel_add_char(char **term_p, const char *src, int *indx)
 791 {
 792     if (src[*indx] == '\\')
 793         *(*term_p)++ = src[(*indx)++];
 794     *(*term_p)++ = src[(*indx)++];
 795 }
 796
 797 /*
 798  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 799  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 800  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 801  *              ([^-a].*|a[^-b].*|ab[c-].*)
 802  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 803  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 804  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 805  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 806  */
 807 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 808                            const char **term_sub, char *term_dict,
 809                            oid_value attributeSet,
 810                            int reg_type, int space_split, char *term_dst,
 811                            int *error_code)
 812 {
 813     AttrType relation;
 814     int relation_value;
 815     int i;
 816     char *term_tmp = term_dict + strlen(term_dict);
 817     char term_component[2*IT_MAX_WORD+20];
 818
 819     attr_init(&relation, zapt, 2);
 820     relation_value = attr_find(&relation, NULL);
 821
 822     *error_code = 0;
 823     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 824     switch (relation_value)
 825     {
 826     case 1:
 827         if (!term_100(zh->reg->zebra_maps, reg_type,
 828                       term_sub, term_component,
 829                       space_split, term_dst))
 830             return 0;
 831         yaz_log(log_level_rpn, "Relation <");
 832
 833         *term_tmp++ = '(';
 834         for (i = 0; term_component[i]; )
 835         {
 836             int j = 0;
 837
 838             if (i)
 839                 *term_tmp++ = '|';
 840             while (j < i)
 841                 string_rel_add_char(&term_tmp, term_component, &j);
 842
 843             *term_tmp++ = '[';
 844
 845             *term_tmp++ = '^';
 846             string_rel_add_char(&term_tmp, term_component, &i);
 847             *term_tmp++ = '-';
 848
 849             *term_tmp++ = ']';
 850             *term_tmp++ = '.';
 851             *term_tmp++ = '*';
 852
 853             if ((term_tmp - term_dict) > IT_MAX_WORD)
 854                 break;
 855         }
 856         *term_tmp++ = ')';
 857         *term_tmp = '\0';
 858         break;
 859     case 2:
 860         if (!term_100(zh->reg->zebra_maps, reg_type,
 861                       term_sub, term_component,
 862                       space_split, term_dst))
 863             return 0;
 864         yaz_log(log_level_rpn, "Relation <=");
 865
 866         *term_tmp++ = '(';
 867         for (i = 0; term_component[i]; )
 868         {
 869             int j = 0;
 870
 871             while (j < i)
 872                 string_rel_add_char(&term_tmp, term_component, &j);
 873             *term_tmp++ = '[';
 874
 875             *term_tmp++ = '^';
 876             string_rel_add_char(&term_tmp, term_component, &i);
 877             *term_tmp++ = '-';
 878
 879             *term_tmp++ = ']';
 880             *term_tmp++ = '.';
 881             *term_tmp++ = '*';
 882
 883             *term_tmp++ = '|';
 884
 885             if ((term_tmp - term_dict) > IT_MAX_WORD)
 886                 break;
 887         }
 888         for (i = 0; term_component[i]; )
 889             string_rel_add_char(&term_tmp, term_component, &i);
 890         *term_tmp++ = ')';
 891         *term_tmp = '\0';
 892         break;
 893     case 5:
 894         if (!term_100 (zh->reg->zebra_maps, reg_type,
 895                        term_sub, term_component, space_split, term_dst))
 896             return 0;
 897         yaz_log(log_level_rpn, "Relation >");
 898
 899         *term_tmp++ = '(';
 900         for (i = 0; term_component[i];)
 901         {
 902             int j = 0;
 903
 904             while (j < i)
 905                 string_rel_add_char(&term_tmp, term_component, &j);
 906             *term_tmp++ = '[';
 907
 908             *term_tmp++ = '^';
 909             *term_tmp++ = '-';
 910             string_rel_add_char(&term_tmp, term_component, &i);
 911
 912             *term_tmp++ = ']';
 913             *term_tmp++ = '.';
 914             *term_tmp++ = '*';
 915
 916             *term_tmp++ = '|';
 917
 918             if ((term_tmp - term_dict) > IT_MAX_WORD)
 919                 break;
 920         }
 921         for (i = 0; term_component[i];)
 922             string_rel_add_char(&term_tmp, term_component, &i);
 923         *term_tmp++ = '.';
 924         *term_tmp++ = '+';
 925         *term_tmp++ = ')';
 926         *term_tmp = '\0';
 927         break;
 928     case 4:
 929         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 930                       term_component, space_split, term_dst))
 931             return 0;
 932         yaz_log(log_level_rpn, "Relation >=");
 933
 934         *term_tmp++ = '(';
 935         for (i = 0; term_component[i];)
 936         {
 937             int j = 0;
 938
 939             if (i)
 940                 *term_tmp++ = '|';
 941             while (j < i)
 942                 string_rel_add_char(&term_tmp, term_component, &j);
 943             *term_tmp++ = '[';
 944
 945             if (term_component[i+1])
 946             {
 947                 *term_tmp++ = '^';
 948                 *term_tmp++ = '-';
 949                 string_rel_add_char(&term_tmp, term_component, &i);
 950             }
 951             else
 952             {
 953                 string_rel_add_char(&term_tmp, term_component, &i);
 954                 *term_tmp++ = '-';
 955             }
 956             *term_tmp++ = ']';
 957             *term_tmp++ = '.';
 958             *term_tmp++ = '*';
 959
 960             if ((term_tmp - term_dict) > IT_MAX_WORD)
 961                 break;
 962         }
 963         *term_tmp++ = ')';
 964         *term_tmp = '\0';
 965         break;
 966     case 3:
 967     case 102:
 968     case -1:
 969         yaz_log(log_level_rpn, "Relation =");
 970         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 971                       term_component, space_split, term_dst))
 972             return 0;
 973         strcat(term_tmp, "(");
 974         strcat(term_tmp, term_component);
 975         strcat(term_tmp, ")");
 976         break;
 977     default:
 978         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 979         return 0;
 980     }
 981     return 1;
 982 }
 983
 984 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 985                              const char **term_sub,
 986                              oid_value attributeSet, NMEM stream,
 987                              struct grep_info *grep_info,
 988                              int reg_type, int complete_flag,
 989                              int num_bases, char **basenames,
 990                              char *term_dst, int xpath_use,
 991                              struct ord_list **ol);
 992
 993 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 994                                  Z_AttributesPlusTerm *zapt,
 995                                  zint *hits_limit_value,
 996                                  const char **term_ref_id_str,
 997                                  NMEM nmem)
 998 {
 999     AttrType term_ref_id_attr;
1000     AttrType hits_limit_attr;
1001     int term_ref_id_int;
1002
1003     attr_init(&hits_limit_attr, zapt, 9);
1004     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
1005
1006     attr_init(&term_ref_id_attr, zapt, 10);
1007     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
1008     if (term_ref_id_int != -1)
1009     {
1010         char *res = nmem_malloc(nmem, 20);
1011         sprintf(res, "%d", term_ref_id_int);
1012         *term_ref_id_str = res;
1013     }
1014
1015     /* no limit given ? */
1016     if (*hits_limit_value == -1)
1017     {
1018         if (*term_ref_id_str)
1019         {
1020             /* use global if term_ref is present */
1021             *hits_limit_value = zh->approx_limit;
1022         }
1023         else
1024         {
1025             /* no counting if term_ref is not present */
1026             *hits_limit_value = 0;
1027         }
1028     }
1029     else if (*hits_limit_value == 0)
1030     {
1031         /* 0 is the same as global limit */
1032         *hits_limit_value = zh->approx_limit;
1033     }
1034     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
1035             *term_ref_id_str ? *term_ref_id_str : "none",
1036             *hits_limit_value);
1037     return ZEBRA_OK;
1038 }
1039
1040 static ZEBRA_RES term_trunc(ZebraHandle zh,
1041                             Z_AttributesPlusTerm *zapt,
1042                             const char **term_sub,
1043                             oid_value attributeSet, NMEM stream,
1044                             struct grep_info *grep_info,
1045                             int reg_type, int complete_flag,
1046                             int num_bases, char **basenames,
1047                             char *term_dst,
1048                             const char *rank_type, int xpath_use,
1049                             NMEM rset_nmem,
1050                             RSET *rset,
1051                             struct rset_key_control *kc)
1052 {
1053     ZEBRA_RES res;
1054     struct ord_list *ol;
1055     zint hits_limit_value;
1056     const char *term_ref_id_str = 0;
1057     *rset = 0;
1058
1059     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str,
1060                     stream);
1061     grep_info->isam_p_indx = 0;
1062     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
1063                       reg_type, complete_flag, num_bases, basenames,
1064                       term_dst, xpath_use, &ol);
1065     if (res != ZEBRA_OK)
1066         return res;
1067     if (!*term_sub)  /* no more terms ? */
1068         return res;
1069     yaz_log(log_level_rpn, "term: %s", term_dst);
1070     *rset = rset_trunc(zh, grep_info->isam_p_buf,
1071                        grep_info->isam_p_indx, term_dst,
1072                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1073                        zapt->term->which, rset_nmem,
1074                        kc, kc->scope, ol, reg_type, hits_limit_value,
1075                        term_ref_id_str);
1076     if (!*rset)
1077         return ZEBRA_FAIL;
1078     return ZEBRA_OK;
1079 }
1080
1081 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1082                              const char **term_sub,
1083                              oid_value attributeSet, NMEM stream,
1084                              struct grep_info *grep_info,
1085                              int reg_type, int complete_flag,
1086                              int num_bases, char **basenames,
1087                              char *term_dst, int xpath_use,
1088                              struct ord_list **ol)
1089 {
1090     char term_dict[2*IT_MAX_WORD+4000];
1091     int j, r, base_no;
1092     AttrType truncation;
1093     int truncation_value;
1094     AttrType use;
1095     int use_value;
1096     const char *use_string = 0;
1097     oid_value curAttributeSet = attributeSet;
1098     const char *termp;
1099     struct rpn_char_map_info rcmi;
1100     int space_split = complete_flag ? 0 : 1;
1101
1102     int bases_ok = 0;     /* no of databases with OK attribute */
1103
1104     *ol = ord_list_create(stream);
1105
1106     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1107     attr_init(&use, zapt, 1);
1108     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1109     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1110     attr_init(&truncation, zapt, 5);
1111     truncation_value = attr_find(&truncation, NULL);
1112     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1113
1114     if (use_value == -1)    /* no attribute - assumy "any" */
1115         use_value = 1016;
1116     for (base_no = 0; base_no < num_bases; base_no++)
1117     {
1118         int ord = -1;
1119         int attr_ok = 0;
1120         int regex_range = 0;
1121         int init_pos = 0;
1122         attent attp;
1123         data1_local_attribute id_xpath_attr;
1124         data1_local_attribute *local_attr;
1125         int max_pos, prefix_len = 0;
1126         int relation_error;
1127
1128         termp = *term_sub;
1129
1130         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1131         {
1132             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1133                            basenames[base_no]);
1134             return ZEBRA_FAIL;
1135         }
1136         if (xpath_use > 0 && use_value == -2)
1137         {
1138             /* xpath mode and we have a string attribute */
1139             attp.local_attributes = &id_xpath_attr;
1140             attp.attset_ordinal = VAL_IDXPATH;
1141             id_xpath_attr.next = 0;
1142
1143             use_value = xpath_use;  /* xpath_use as use-attribute now */
1144             id_xpath_attr.local = use_value;
1145         }
1146         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1147         {
1148             /* X-Path attribute, use numeric value directly */
1149             attp.local_attributes = &id_xpath_attr;
1150             attp.attset_ordinal = VAL_IDXPATH;
1151             id_xpath_attr.next = 0;
1152             id_xpath_attr.local = use_value;
1153         }
1154         else if (use_string &&
1155                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1156                                                      reg_type,
1157                                                      use_string)) >= 0)
1158         {
1159             /* we have a match for a raw string attribute */
1160             char ord_buf[32];
1161             int i, ord_len;
1162
1163             if (prefix_len)
1164                 term_dict[prefix_len++] = '|';
1165             else
1166                 term_dict[prefix_len++] = '(';
1167
1168             ord_len = key_SU_encode (ord, ord_buf);
1169             for (i = 0; i<ord_len; i++)
1170             {
1171                 term_dict[prefix_len++] = 1;
1172                 term_dict[prefix_len++] = ord_buf[i];
1173             }
1174             attp.local_attributes = 0;  /* no more attributes */
1175             *ol = ord_list_append(stream, *ol, ord);
1176         }
1177         else
1178         {
1179             /* lookup in the .att files . Allow string as well */
1180             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1181                                       use_string)))
1182             {
1183                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1184                         curAttributeSet, use_value, r);
1185                 if (r == -1)
1186                 {
1187                     /* set was found, but value wasn't defined */
1188                     if (use_string)
1189                         zebra_setError(zh,
1190                                        YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1191                                        use_string);
1192                     else
1193                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1194                                             use_value);
1195                 }
1196                 else
1197                 {
1198                     int oid[OID_SIZE];
1199                     struct oident oident;
1200
1201                     oident.proto = PROTO_Z3950;
1202                     oident.oclass = CLASS_ATTSET;
1203                     oident.value = curAttributeSet;
1204                     oid_ent_to_oid (&oident, oid);
1205
1206                     zebra_setError(zh,
1207                                    YAZ_BIB1_UNSUPP_ATTRIBUTE_SET,
1208                                    oident.desc);
1209
1210                 }
1211                 continue;
1212             }
1213         }
1214         for (local_attr = attp.local_attributes; local_attr;
1215              local_attr = local_attr->next)
1216         {
1217             char ord_buf[32];
1218             int i, ord_len;
1219
1220             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1221                                               reg_type,
1222                                               attp.attset_ordinal,
1223                                               local_attr->local);
1224             if (ord < 0)
1225                 continue;
1226             *ol = ord_list_append(stream, *ol, ord);
1227             if (prefix_len)
1228                 term_dict[prefix_len++] = '|';
1229             else
1230                 term_dict[prefix_len++] = '(';
1231
1232             ord_len = key_SU_encode (ord, ord_buf);
1233             for (i = 0; i<ord_len; i++)
1234             {
1235                 term_dict[prefix_len++] = 1;
1236                 term_dict[prefix_len++] = ord_buf[i];
1237             }
1238         }
1239         bases_ok++;
1240         if (prefix_len)
1241             attr_ok = 1;
1242
1243         term_dict[prefix_len++] = ')';
1244 #if REG_TYPE_PREFIX
1245         term_dict[prefix_len++] = 1;
1246         term_dict[prefix_len++] = reg_type;
1247         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1248 #endif
1249         term_dict[prefix_len] = '\0';
1250         j = prefix_len;
1251         switch (truncation_value)
1252         {
1253         case -1:         /* not specified */
1254         case 100:        /* do not truncate */
1255             if (!string_relation (zh, zapt, &termp, term_dict,
1256                                   attributeSet,
1257                                   reg_type, space_split, term_dst,
1258                                   &relation_error))
1259             {
1260                 if (relation_error)
1261                 {
1262                     zebra_setError(zh, relation_error, 0);
1263                     return ZEBRA_FAIL;
1264                 }
1265                 *term_sub = 0;
1266                 return ZEBRA_OK;
1267             }
1268             break;
1269         case 1:          /* right truncation */
1270             term_dict[j++] = '(';
1271             if (!term_100(zh->reg->zebra_maps, reg_type,
1272                           &termp, term_dict + j, space_split, term_dst))
1273             {
1274                 *term_sub = 0;
1275                 return ZEBRA_OK;
1276             }
1277             strcat(term_dict, ".*)");
1278             break;
1279         case 2:          /* keft truncation */
1280             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1281             if (!term_100(zh->reg->zebra_maps, reg_type,
1282                           &termp, term_dict + j, space_split, term_dst))
1283             {
1284                 *term_sub = 0;
1285                 return ZEBRA_OK;
1286             }
1287             strcat(term_dict, ")");
1288             break;
1289         case 3:          /* left&right truncation */
1290             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1291             if (!term_100(zh->reg->zebra_maps, reg_type,
1292                           &termp, term_dict + j, space_split, term_dst))
1293             {
1294                 *term_sub = 0;
1295                 return ZEBRA_OK;
1296             }
1297             strcat(term_dict, ".*)");
1298             break;
1299         case 101:        /* process # in term */
1300             term_dict[j++] = '(';
1301             if (!term_101(zh->reg->zebra_maps, reg_type,
1302                           &termp, term_dict + j, space_split, term_dst))
1303             {
1304                 *term_sub = 0;
1305                 return ZEBRA_OK;
1306             }
1307             strcat(term_dict, ")");
1308             break;
1309         case 102:        /* Regexp-1 */
1310             term_dict[j++] = '(';
1311             if (!term_102(zh->reg->zebra_maps, reg_type,
1312                           &termp, term_dict + j, space_split, term_dst))
1313             {
1314                 *term_sub = 0;
1315                 return ZEBRA_OK;
1316             }
1317             strcat(term_dict, ")");
1318             break;
1319         case 103:       /* Regexp-2 */
1320             regex_range = 1;
1321             term_dict[j++] = '(';
1322             init_pos = 2;
1323             if (!term_103(zh->reg->zebra_maps, reg_type,
1324                           &termp, term_dict + j, &regex_range,
1325                           space_split, term_dst))
1326             {
1327                 *term_sub = 0;
1328                 return ZEBRA_OK;
1329             }
1330             strcat(term_dict, ")");
1331             break;
1332         case 104:        /* process # and ! in term */
1333             term_dict[j++] = '(';
1334             if (!term_104(zh->reg->zebra_maps, reg_type,
1335                           &termp, term_dict + j, space_split, term_dst))
1336             {
1337                 *term_sub = 0;
1338                 return ZEBRA_OK;
1339             }
1340             strcat(term_dict, ")");
1341             break;
1342         case 105:        /* process * and ! in term */
1343             term_dict[j++] = '(';
1344             if (!term_105(zh->reg->zebra_maps, reg_type,
1345                           &termp, term_dict + j, space_split, term_dst, 1))
1346             {
1347                 *term_sub = 0;
1348                 return ZEBRA_OK;
1349             }
1350             strcat(term_dict, ")");
1351             break;
1352         case 106:        /* process * and ! in term */
1353             term_dict[j++] = '(';
1354             if (!term_105(zh->reg->zebra_maps, reg_type,
1355                           &termp, term_dict + j, space_split, term_dst, 0))
1356             {
1357                 *term_sub = 0;
1358                 return ZEBRA_OK;
1359             }
1360             strcat(term_dict, ")");
1361             break;
1362         default:
1363             zebra_setError_zint(zh,
1364                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1365                                 truncation_value);
1366             return ZEBRA_FAIL;
1367         }
1368         if (attr_ok)
1369         {
1370             char buf[80];
1371             const char *input = term_dict + prefix_len;
1372             esc_str(buf, sizeof(buf), input, strlen(input));
1373         }
1374         if (attr_ok)
1375         {
1376             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1377             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1378                                  grep_info, &max_pos, init_pos,
1379                                  grep_handle);
1380             if (r)
1381                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1382         }
1383     }
1384     if (!bases_ok)
1385         return ZEBRA_FAIL;
1386     *term_sub = termp;
1387     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1388     return ZEBRA_OK;
1389 }
1390
1391
1392 /* convert APT search term to UTF8 */
1393 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1394                                    char *termz)
1395 {
1396     size_t sizez;
1397     Z_Term *term = zapt->term;
1398
1399     switch (term->which)
1400     {
1401     case Z_Term_general:
1402         if (zh->iconv_to_utf8 != 0)
1403         {
1404             char *inbuf = (char *) term->u.general->buf;
1405             size_t inleft = term->u.general->len;
1406             char *outbuf = termz;
1407             size_t outleft = IT_MAX_WORD-1;
1408             size_t ret;
1409
1410             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1411                         &outbuf, &outleft);
1412             if (ret == (size_t)(-1))
1413             {
1414                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1415                 zebra_setError(
1416                     zh,
1417                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1418                     0);
1419                 return ZEBRA_FAIL;
1420             }
1421             *outbuf = 0;
1422         }
1423         else
1424         {
1425             sizez = term->u.general->len;
1426             if (sizez > IT_MAX_WORD-1)
1427                 sizez = IT_MAX_WORD-1;
1428             memcpy (termz, term->u.general->buf, sizez);
1429             termz[sizez] = '\0';
1430         }
1431         break;
1432     case Z_Term_characterString:
1433         sizez = strlen(term->u.characterString);
1434         if (sizez > IT_MAX_WORD-1)
1435             sizez = IT_MAX_WORD-1;
1436         memcpy (termz, term->u.characterString, sizez);
1437         termz[sizez] = '\0';
1438         break;
1439     default:
1440         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1441         return ZEBRA_FAIL;
1442     }
1443     return ZEBRA_OK;
1444 }
1445
1446 /* convert APT SCAN term to internal cmap */
1447 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1448                                  char *termz, int reg_type)
1449 {
1450     char termz0[IT_MAX_WORD];
1451
1452     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1453         return ZEBRA_FAIL;    /* error */
1454     else
1455     {
1456         const char **map;
1457         const char *cp = (const char *) termz0;
1458         const char *cp_end = cp + strlen(cp);
1459         const char *src;
1460         int i = 0;
1461         const char *space_map = NULL;
1462         int len;
1463
1464         while ((len = (cp_end - cp)) > 0)
1465         {
1466             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1467             if (**map == *CHR_SPACE)
1468                 space_map = *map;
1469             else
1470             {
1471                 if (i && space_map)
1472                     for (src = space_map; *src; src++)
1473                         termz[i++] = *src;
1474                 space_map = NULL;
1475                 for (src = *map; *src; src++)
1476                     termz[i++] = *src;
1477             }
1478         }
1479         termz[i] = '\0';
1480     }
1481     return ZEBRA_OK;
1482 }
1483
1484 static void grep_info_delete(struct grep_info *grep_info)
1485 {
1486 #ifdef TERM_COUNT
1487     xfree(grep_info->term_no);
1488 #endif
1489     xfree(grep_info->isam_p_buf);
1490 }
1491
1492 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1493                                    Z_AttributesPlusTerm *zapt,
1494                                    struct grep_info *grep_info,
1495                                    int reg_type)
1496 {
1497     AttrType termset;
1498     int termset_value_numeric;
1499     const char *termset_value_string;
1500
1501 #ifdef TERM_COUNT
1502     grep_info->term_no = 0;
1503 #endif
1504     grep_info->isam_p_size = 0;
1505     grep_info->isam_p_buf = NULL;
1506     grep_info->zh = zh;
1507     grep_info->reg_type = reg_type;
1508     grep_info->termset = 0;
1509
1510     if (!zapt)
1511         return ZEBRA_OK;
1512     attr_init(&termset, zapt, 8);
1513     termset_value_numeric =
1514         attr_find_ex(&termset, NULL, &termset_value_string);
1515     if (termset_value_numeric != -1)
1516     {
1517         char resname[32];
1518         const char *termset_name = 0;
1519         if (termset_value_numeric != -2)
1520         {
1521
1522             sprintf(resname, "%d", termset_value_numeric);
1523             termset_name = resname;
1524         }
1525         else
1526             termset_name = termset_value_string;
1527         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1528         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1529         if (!grep_info->termset)
1530         {
1531             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1532             return ZEBRA_FAIL;
1533         }
1534     }
1535     return ZEBRA_OK;
1536 }
1537
1538 /**
1539   \brief Create result set(s) for list of terms
1540   \param zh Zebra Handle
1541   \param termz term as used in query but converted to UTF-8
1542   \param attributeSet default attribute set
1543   \param stream memory for result
1544   \param reg_type register type ('w', 'p',..)
1545   \param complete_flag whether it's phrases or not
1546   \param rank_type term flags for ranking
1547   \param xpath_use use attribute for X-Path (-1 for no X-path)
1548   \param num_bases number of databases
1549   \param basenames array of databases
1550   \param rset_mem memory for result sets
1551   \param result_sets output result set for each term in list (output)
1552   \param number number of output result sets
1553   \param kc rset key control to be used for created result sets
1554 */
1555 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1556                                  Z_AttributesPlusTerm *zapt,
1557                                  const char *termz,
1558                                  oid_value attributeSet,
1559                                  NMEM stream,
1560                                  int reg_type, int complete_flag,
1561                                  const char *rank_type, int xpath_use,
1562                                  int num_bases, char **basenames,
1563                                  NMEM rset_nmem,
1564                                  RSET **result_sets, int *num_result_sets,
1565                                  struct rset_key_control *kc)
1566 {
1567     char term_dst[IT_MAX_WORD+1];
1568     struct grep_info grep_info;
1569     const char *termp = termz;
1570     int alloc_sets = 0;
1571
1572     *num_result_sets = 0;
1573     *term_dst = 0;
1574     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1575         return ZEBRA_FAIL;
1576     while(1)
1577     {
1578         ZEBRA_RES res;
1579
1580         if (alloc_sets == *num_result_sets)
1581         {
1582             int add = 10;
1583             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1584                                               sizeof(*rnew));
1585             if (alloc_sets)
1586                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1587             alloc_sets = alloc_sets + add;
1588             *result_sets = rnew;
1589         }
1590         res = term_trunc(zh, zapt, &termp, attributeSet,
1591                          stream, &grep_info,
1592                          reg_type, complete_flag,
1593                          num_bases, basenames,
1594                          term_dst, rank_type,
1595                          xpath_use, rset_nmem,
1596                          &(*result_sets)[*num_result_sets],
1597                          kc);
1598         if (res != ZEBRA_OK)
1599         {
1600             int i;
1601             for (i = 0; i < *num_result_sets; i++)
1602                 rset_delete((*result_sets)[i]);
1603             grep_info_delete (&grep_info);
1604             return res;
1605         }
1606         if ((*result_sets)[*num_result_sets] == 0)
1607             break;
1608         (*num_result_sets)++;
1609     }
1610     grep_info_delete(&grep_info);
1611     return ZEBRA_OK;
1612 }
1613
1614 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1615                                        Z_AttributesPlusTerm *zapt,
1616                                        const char *termz_org,
1617                                        oid_value attributeSet,
1618                                        NMEM stream,
1619                                        int reg_type, int complete_flag,
1620                                        const char *rank_type, int xpath_use,
1621                                        int num_bases, char **basenames,
1622                                        NMEM rset_nmem,
1623                                        RSET *rset,
1624                                        struct rset_key_control *kc)
1625 {
1626     RSET *result_sets = 0;
1627     int num_result_sets = 0;
1628     ZEBRA_RES res =
1629         term_list_trunc(zh, zapt, termz_org, attributeSet,
1630                         stream, reg_type, complete_flag,
1631                         rank_type, xpath_use,
1632                         num_bases, basenames,
1633                         rset_nmem,
1634                         &result_sets, &num_result_sets, kc);
1635     if (res != ZEBRA_OK)
1636         return res;
1637     if (num_result_sets == 0)
1638         *rset = rsnull_create (rset_nmem, kc, 0);
1639     else if (num_result_sets == 1)
1640         *rset = result_sets[0];
1641     else
1642         *rset = rsprox_create(rset_nmem, kc, kc->scope,
1643                               num_result_sets, result_sets,
1644                               1 /* ordered */, 0 /* exclusion */,
1645                               3 /* relation */, 1 /* distance */);
1646     if (!*rset)
1647         return ZEBRA_FAIL;
1648     return ZEBRA_OK;
1649 }
1650
1651 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1652                                         Z_AttributesPlusTerm *zapt,
1653                                         const char *termz_org,
1654                                         oid_value attributeSet,
1655                                         NMEM stream,
1656                                         int reg_type, int complete_flag,
1657                                         const char *rank_type,
1658                                         int xpath_use,
1659                                         int num_bases, char **basenames,
1660                                         NMEM rset_nmem,
1661                                         RSET *rset,
1662                                         struct rset_key_control *kc)
1663 {
1664     RSET *result_sets = 0;
1665     int num_result_sets = 0;
1666     ZEBRA_RES res =
1667         term_list_trunc(zh, zapt, termz_org, attributeSet,
1668                         stream, reg_type, complete_flag,
1669                         rank_type, xpath_use,
1670                         num_bases, basenames,
1671                         rset_nmem,
1672                         &result_sets, &num_result_sets, kc);
1673     if (res != ZEBRA_OK)
1674         return res;
1675     if (num_result_sets == 0)
1676         *rset = rsnull_create (rset_nmem, kc, 0);
1677     else if (num_result_sets == 1)
1678         *rset = result_sets[0];
1679     else
1680         *rset = rsmulti_or_create(rset_nmem, kc, kc->scope, 0 /* termid */,
1681                                   num_result_sets, result_sets);
1682     if (!*rset)
1683         return ZEBRA_FAIL;
1684     return ZEBRA_OK;
1685 }
1686
1687 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1688                                          Z_AttributesPlusTerm *zapt,
1689                                          const char *termz_org,
1690                                          oid_value attributeSet,
1691                                          NMEM stream,
1692                                          int reg_type, int complete_flag,
1693                                          const char *rank_type,
1694                                          int xpath_use,
1695                                          int num_bases, char **basenames,
1696                                          NMEM rset_nmem,
1697                                          RSET *rset,
1698                                          struct rset_key_control *kc)
1699 {
1700     RSET *result_sets = 0;
1701     int num_result_sets = 0;
1702     ZEBRA_RES res =
1703         term_list_trunc(zh, zapt, termz_org, attributeSet,
1704                         stream, reg_type, complete_flag,
1705                         rank_type, xpath_use,
1706                         num_bases, basenames,
1707                         rset_nmem,
1708                         &result_sets, &num_result_sets,
1709                         kc);
1710     if (res != ZEBRA_OK)
1711         return res;
1712     if (num_result_sets == 0)
1713         *rset = rsnull_create (rset_nmem, kc, 0);
1714     else if (num_result_sets == 1)
1715         *rset = result_sets[0];
1716     else
1717         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1718                                    num_result_sets, result_sets);
1719     if (!*rset)
1720         return ZEBRA_FAIL;
1721     return ZEBRA_OK;
1722 }
1723
1724 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1725                             const char **term_sub,
1726                             char *term_dict,
1727                             oid_value attributeSet,
1728                             struct grep_info *grep_info,
1729                             int *max_pos,
1730                             int reg_type,
1731                             char *term_dst,
1732                             int *error_code)
1733 {
1734     AttrType relation;
1735     int relation_value;
1736     int term_value;
1737     int r;
1738     char *term_tmp = term_dict + strlen(term_dict);
1739
1740     *error_code = 0;
1741     attr_init(&relation, zapt, 2);
1742     relation_value = attr_find(&relation, NULL);
1743
1744     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1745
1746     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1747                   term_dst))
1748         return 0;
1749     term_value = atoi (term_tmp);
1750     switch (relation_value)
1751     {
1752     case 1:
1753         yaz_log(log_level_rpn, "Relation <");
1754         gen_regular_rel(term_tmp, term_value-1, 1);
1755         break;
1756     case 2:
1757         yaz_log(log_level_rpn, "Relation <=");
1758         gen_regular_rel(term_tmp, term_value, 1);
1759         break;
1760     case 4:
1761         yaz_log(log_level_rpn, "Relation >=");
1762         gen_regular_rel(term_tmp, term_value, 0);
1763         break;
1764     case 5:
1765         yaz_log(log_level_rpn, "Relation >");
1766         gen_regular_rel(term_tmp, term_value+1, 0);
1767         break;
1768     case -1:
1769     case 3:
1770         yaz_log(log_level_rpn, "Relation =");
1771         sprintf(term_tmp, "(0*%d)", term_value);
1772         break;
1773     default:
1774         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1775         return 0;
1776     }
1777     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1778     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1779                           0, grep_handle);
1780     if (r)
1781         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1782     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1783     return 1;
1784 }
1785
1786 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1787                               const char **term_sub,
1788                               oid_value attributeSet,
1789                               struct grep_info *grep_info,
1790                               int reg_type, int complete_flag,
1791                               int num_bases, char **basenames,
1792                               char *term_dst, int xpath_use, NMEM stream)
1793 {
1794     char term_dict[2*IT_MAX_WORD+2];
1795     int r, base_no;
1796     AttrType use;
1797     int use_value;
1798     const char *use_string = 0;
1799     oid_value curAttributeSet = attributeSet;
1800     const char *termp;
1801     struct rpn_char_map_info rcmi;
1802
1803     int bases_ok = 0;     /* no of databases with OK attribute */
1804
1805     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1806     attr_init(&use, zapt, 1);
1807     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1808
1809     if (use_value == -1)
1810         use_value = 1016;
1811
1812     for (base_no = 0; base_no < num_bases; base_no++)
1813     {
1814         attent attp;
1815         data1_local_attribute id_xpath_attr;
1816         data1_local_attribute *local_attr;
1817         int max_pos, prefix_len = 0;
1818         int relation_error = 0;
1819
1820         termp = *term_sub;
1821         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1822         {
1823             use_value = xpath_use;
1824             attp.local_attributes = &id_xpath_attr;
1825             attp.attset_ordinal = VAL_IDXPATH;
1826             id_xpath_attr.next = 0;
1827             id_xpath_attr.local = use_value;
1828         }
1829         else if (curAttributeSet == VAL_IDXPATH)
1830         {
1831             attp.local_attributes = &id_xpath_attr;
1832             attp.attset_ordinal = VAL_IDXPATH;
1833             id_xpath_attr.next = 0;
1834             id_xpath_attr.local = use_value;
1835         }
1836         else
1837         {
1838             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1839                                       use_string)))
1840             {
1841                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1842                       curAttributeSet, use_value, r);
1843                 if (r == -1)
1844                 {
1845                     if (use_string)
1846                         zebra_setError(zh,
1847                                        YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1848                                        use_string);
1849                     else
1850                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1851                                             use_value);
1852                 }
1853                 else
1854                     zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_SET, 0);
1855                 continue;
1856             }
1857         }
1858         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1859         {
1860             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1861                            basenames[base_no]);
1862             return ZEBRA_FAIL;
1863         }
1864         for (local_attr = attp.local_attributes; local_attr;
1865              local_attr = local_attr->next)
1866         {
1867             int ord;
1868             char ord_buf[32];
1869             int i, ord_len;
1870
1871             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1872                                               reg_type,
1873                                               attp.attset_ordinal,
1874                                               local_attr->local);
1875             if (ord < 0)
1876                 continue;
1877             if (prefix_len)
1878                 term_dict[prefix_len++] = '|';
1879             else
1880                 term_dict[prefix_len++] = '(';
1881
1882             ord_len = key_SU_encode (ord, ord_buf);
1883             for (i = 0; i<ord_len; i++)
1884             {
1885                 term_dict[prefix_len++] = 1;
1886                 term_dict[prefix_len++] = ord_buf[i];
1887             }
1888         }
1889         if (!prefix_len)
1890         {
1891             zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE, use_value);
1892             continue;
1893         }
1894         bases_ok++;
1895         term_dict[prefix_len++] = ')';
1896 #if REG_TYPE_PREFIX
1897         term_dict[prefix_len++] = 1;
1898         term_dict[prefix_len++] = reg_type;
1899         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1900 #endif
1901         term_dict[prefix_len] = '\0';
1902         if (!numeric_relation(zh, zapt, &termp, term_dict,
1903                               attributeSet, grep_info, &max_pos, reg_type,
1904                               term_dst, &relation_error))
1905         {
1906             if (relation_error)
1907             {
1908                 zebra_setError(zh, relation_error, 0);
1909                 return ZEBRA_FAIL;
1910             }
1911             *term_sub = 0;
1912             return ZEBRA_OK;
1913         }
1914     }
1915     if (!bases_ok)
1916         return ZEBRA_FAIL;
1917     *term_sub = termp;
1918     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1919     return ZEBRA_OK;
1920 }
1921
1922
1923 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1924                                         Z_AttributesPlusTerm *zapt,
1925                                         const char *termz,
1926                                         oid_value attributeSet,
1927                                         NMEM stream,
1928                                         int reg_type, int complete_flag,
1929                                         const char *rank_type, int xpath_use,
1930                                         int num_bases, char **basenames,
1931                                         NMEM rset_nmem,
1932                                         RSET *rset,
1933                                         struct rset_key_control *kc)
1934 {
1935     char term_dst[IT_MAX_WORD+1];
1936     const char *termp = termz;
1937     RSET *result_sets = 0;
1938     int num_result_sets = 0;
1939     ZEBRA_RES res;
1940     struct grep_info grep_info;
1941     int alloc_sets = 0;
1942     zint hits_limit_value;
1943     const char *term_ref_id_str = 0;
1944
1945     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1946
1947     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1948     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1949         return ZEBRA_FAIL;
1950     while (1)
1951     {
1952         if (alloc_sets == num_result_sets)
1953         {
1954             int add = 10;
1955             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1956                                               sizeof(*rnew));
1957             if (alloc_sets)
1958                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1959             alloc_sets = alloc_sets + add;
1960             result_sets = rnew;
1961         }
1962         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1963         grep_info.isam_p_indx = 0;
1964         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1965                            reg_type, complete_flag, num_bases, basenames,
1966                            term_dst, xpath_use,
1967                            stream);
1968         if (res == ZEBRA_FAIL || termp == 0)
1969             break;
1970         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1971         result_sets[num_result_sets] =
1972             rset_trunc(zh, grep_info.isam_p_buf,
1973                        grep_info.isam_p_indx, term_dst,
1974                        strlen(term_dst), rank_type,
1975                        0 /* preserve position */,
1976                        zapt->term->which, rset_nmem,
1977                        kc, kc->scope, 0, reg_type,
1978                        hits_limit_value,
1979                        term_ref_id_str);
1980         if (!result_sets[num_result_sets])
1981             break;
1982         num_result_sets++;
1983     }
1984     grep_info_delete(&grep_info);
1985     if (termp)
1986     {
1987         int i;
1988         for (i = 0; i<num_result_sets; i++)
1989             rset_delete(result_sets[i]);
1990         return ZEBRA_FAIL;
1991     }
1992     if (num_result_sets == 0)
1993         *rset = rsnull_create(rset_nmem, kc, 0);
1994     if (num_result_sets == 1)
1995         *rset = result_sets[0];
1996     else
1997         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1998                                    num_result_sets, result_sets);
1999     if (!*rset)
2000         return ZEBRA_FAIL;
2001     return ZEBRA_OK;
2002 }
2003
2004 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
2005                                       Z_AttributesPlusTerm *zapt,
2006                                       const char *termz,
2007                                       oid_value attributeSet,
2008                                       NMEM stream,
2009                                       const char *rank_type, NMEM rset_nmem,
2010                                       RSET *rset,
2011                                       struct rset_key_control *kc)
2012 {
2013     RSFD rsfd;
2014     struct it_key key;
2015     int sys;
2016     *rset = rstemp_create(rset_nmem, kc, kc->scope,
2017                           res_get (zh->res, "setTmpDir"),0 );
2018     rsfd = rset_open(*rset, RSETF_WRITE);
2019
2020     sys = atoi(termz);
2021     if (sys <= 0)
2022         sys = 1;
2023     key.mem[0] = sys;
2024     key.mem[1] = 1;
2025     key.len = 2;
2026     rset_write (rsfd, &key);
2027     rset_close (rsfd);
2028     return ZEBRA_OK;
2029 }
2030
2031 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2032                                oid_value attributeSet, NMEM stream,
2033                                Z_SortKeySpecList *sort_sequence,
2034                                const char *rank_type,
2035                                NMEM rset_nmem,
2036                                RSET *rset,
2037                                struct rset_key_control *kc)
2038 {
2039     int i;
2040     int sort_relation_value;
2041     AttrType sort_relation_type;
2042     Z_SortKeySpec *sks;
2043     Z_SortKey *sk;
2044     int oid[OID_SIZE];
2045     oident oe;
2046     char termz[20];
2047
2048     attr_init(&sort_relation_type, zapt, 7);
2049     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
2050
2051     if (!sort_sequence->specs)
2052     {
2053         sort_sequence->num_specs = 10;
2054         sort_sequence->specs = (Z_SortKeySpec **)
2055             nmem_malloc(stream, sort_sequence->num_specs *
2056                          sizeof(*sort_sequence->specs));
2057         for (i = 0; i<sort_sequence->num_specs; i++)
2058             sort_sequence->specs[i] = 0;
2059     }
2060     if (zapt->term->which != Z_Term_general)
2061         i = 0;
2062     else
2063         i = atoi_n ((char *) zapt->term->u.general->buf,
2064                     zapt->term->u.general->len);
2065     if (i >= sort_sequence->num_specs)
2066         i = 0;
2067     sprintf(termz, "%d", i);
2068
2069     oe.proto = PROTO_Z3950;
2070     oe.oclass = CLASS_ATTSET;
2071     oe.value = attributeSet;
2072     if (!oid_ent_to_oid (&oe, oid))
2073         return ZEBRA_FAIL;
2074
2075     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2076     sks->sortElement = (Z_SortElement *)
2077         nmem_malloc(stream, sizeof(*sks->sortElement));
2078     sks->sortElement->which = Z_SortElement_generic;
2079     sk = sks->sortElement->u.generic = (Z_SortKey *)
2080         nmem_malloc(stream, sizeof(*sk));
2081     sk->which = Z_SortKey_sortAttributes;
2082     sk->u.sortAttributes = (Z_SortAttributes *)
2083         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2084
2085     sk->u.sortAttributes->id = oid;
2086     sk->u.sortAttributes->list = zapt->attributes;
2087
2088     sks->sortRelation = (int *)
2089         nmem_malloc(stream, sizeof(*sks->sortRelation));
2090     if (sort_relation_value == 1)
2091         *sks->sortRelation = Z_SortKeySpec_ascending;
2092     else if (sort_relation_value == 2)
2093         *sks->sortRelation = Z_SortKeySpec_descending;
2094     else
2095         *sks->sortRelation = Z_SortKeySpec_ascending;
2096
2097     sks->caseSensitivity = (int *)
2098         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2099     *sks->caseSensitivity = 0;
2100
2101     sks->which = Z_SortKeySpec_null;
2102     sks->u.null = odr_nullval ();
2103     sort_sequence->specs[i] = sks;
2104     *rset = rsnull_create (rset_nmem, kc, 0);
2105     return ZEBRA_OK;
2106 }
2107
2108
2109 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2110                        oid_value attributeSet,
2111                        struct xpath_location_step *xpath, int max, NMEM mem)
2112 {
2113     oid_value curAttributeSet = attributeSet;
2114     AttrType use;
2115     const char *use_string = 0;
2116
2117     attr_init(&use, zapt, 1);
2118     attr_find_ex(&use, &curAttributeSet, &use_string);
2119
2120     if (!use_string || *use_string != '/')
2121         return -1;
2122
2123     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2124 }
2125
2126
2127
2128 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2129                         int reg_type, const char *term, int use,
2130                         oid_value curAttributeSet, NMEM rset_nmem,
2131                         struct rset_key_control *kc)
2132 {
2133     RSET rset;
2134     struct grep_info grep_info;
2135     char term_dict[2048];
2136     char ord_buf[32];
2137     int prefix_len = 0;
2138     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, reg_type,
2139                                           curAttributeSet, use);
2140     int ord_len, i, r, max_pos;
2141     int term_type = Z_Term_characterString;
2142     const char *flags = "void";
2143
2144     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
2145         return rsnull_create(rset_nmem, kc, 0);
2146
2147     if (ord < 0)
2148         return rsnull_create(rset_nmem, kc, 0);
2149     if (prefix_len)
2150         term_dict[prefix_len++] = '|';
2151     else
2152         term_dict[prefix_len++] = '(';
2153
2154     ord_len = key_SU_encode (ord, ord_buf);
2155     for (i = 0; i<ord_len; i++)
2156     {
2157         term_dict[prefix_len++] = 1;
2158         term_dict[prefix_len++] = ord_buf[i];
2159     }
2160     term_dict[prefix_len++] = ')';
2161 #if REG_TYPE_PREFIX
2162     term_dict[prefix_len++] = 1;
2163     term_dict[prefix_len++] = reg_type;
2164 #endif
2165     strcpy(term_dict+prefix_len, term);
2166
2167     grep_info.isam_p_indx = 0;
2168     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2169                           &grep_info, &max_pos, 0, grep_handle);
2170     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2171              grep_info.isam_p_indx);
2172     rset = rset_trunc(zh, grep_info.isam_p_buf,
2173                       grep_info.isam_p_indx, term, strlen(term),
2174                       flags, 1, term_type,rset_nmem,
2175                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
2176                       0 /* term_ref_id_str */);
2177     grep_info_delete(&grep_info);
2178     return rset;
2179 }
2180
2181 static
2182 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
2183                            oid_value attributeSet,
2184                            int num_bases, char **basenames,
2185                            NMEM stream, const char *rank_type, RSET rset,
2186                            int xpath_len, struct xpath_location_step *xpath,
2187                            NMEM rset_nmem,
2188                            RSET *rset_out,
2189                            struct rset_key_control *kc)
2190 {
2191     oid_value curAttributeSet = attributeSet;
2192     int base_no;
2193     int i;
2194
2195     if (xpath_len < 0)
2196     {
2197         *rset_out = rset;
2198         return ZEBRA_OK;
2199     }
2200
2201     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2202     for (i = 0; i<xpath_len; i++)
2203     {
2204         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2205
2206     }
2207
2208     curAttributeSet = VAL_IDXPATH;
2209
2210     /*
2211       //a    ->    a/.*
2212       //a/b  ->    b/a/.*
2213       /a     ->    a/
2214       /a/b   ->    b/a/
2215
2216       /      ->    none
2217
2218    a[@attr = value]/b[@other = othervalue]
2219
2220  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2221  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2222  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2223  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2224  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2225  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2226
2227     */
2228
2229     dict_grep_cmap (zh->reg->dict, 0, 0);
2230
2231     for (base_no = 0; base_no < num_bases; base_no++)
2232     {
2233         int level = xpath_len;
2234         int first_path = 1;
2235
2236         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2237         {
2238             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2239                            basenames[base_no]);
2240             *rset_out = rset;
2241             return ZEBRA_FAIL;
2242         }
2243         while (--level >= 0)
2244         {
2245             char xpath_rev[128];
2246             int i, len;
2247             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2248
2249             *xpath_rev = 0;
2250             len = 0;
2251             for (i = level; i >= 1; --i)
2252             {
2253                 const char *cp = xpath[i].part;
2254                 if (*cp)
2255                 {
2256                     for (;*cp; cp++)
2257                         if (*cp == '*')
2258                         {
2259                             memcpy (xpath_rev + len, "[^/]*", 5);
2260                             len += 5;
2261                         }
2262                         else if (*cp == ' ')
2263                         {
2264
2265                             xpath_rev[len++] = 1;
2266                             xpath_rev[len++] = ' ';
2267                         }
2268
2269                         else
2270                             xpath_rev[len++] = *cp;
2271                     xpath_rev[len++] = '/';
2272                 }
2273                 else if (i == 1)  /* // case */
2274                 {
2275                     xpath_rev[len++] = '.';
2276                     xpath_rev[len++] = '*';
2277                 }
2278             }
2279             xpath_rev[len] = 0;
2280
2281             if (xpath[level].predicate &&
2282                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2283                 xpath[level].predicate->u.relation.name[0])
2284             {
2285                 WRBUF wbuf = wrbuf_alloc();
2286                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2287                 if (xpath[level].predicate->u.relation.value)
2288                 {
2289                     const char *cp = xpath[level].predicate->u.relation.value;
2290                     wrbuf_putc(wbuf, '=');
2291
2292                     while (*cp)
2293                     {
2294                         if (strchr(REGEX_CHARS, *cp))
2295                             wrbuf_putc(wbuf, '\\');
2296                         wrbuf_putc(wbuf, *cp);
2297                         cp++;
2298                     }
2299                 }
2300                 wrbuf_puts(wbuf, "");
2301                 rset_attr = xpath_trunc(
2302                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2303                     curAttributeSet, rset_nmem, kc);
2304                 wrbuf_free(wbuf, 1);
2305             }
2306             else
2307             {
2308                 if (!first_path)
2309                     continue;
2310             }
2311             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2312             if (strlen(xpath_rev))
2313             {
2314                 rset_start_tag = xpath_trunc(zh, stream, '0',
2315                         xpath_rev, 1, curAttributeSet, rset_nmem, kc);
2316
2317                 rset_end_tag = xpath_trunc(zh, stream, '0',
2318                         xpath_rev, 2, curAttributeSet, rset_nmem, kc);
2319
2320                 rset = rsbetween_create(rset_nmem, kc, kc->scope,
2321                                         rset_start_tag, rset,
2322                                         rset_end_tag, rset_attr);
2323             }
2324             first_path = 0;
2325         }
2326     }
2327     *rset_out = rset;
2328     return ZEBRA_OK;
2329 }
2330
2331 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2332                                 oid_value attributeSet, NMEM stream,
2333                                 Z_SortKeySpecList *sort_sequence,
2334                                 int num_bases, char **basenames,
2335                                 NMEM rset_nmem,
2336                                 RSET *rset,
2337                                 struct rset_key_control *kc)
2338 {
2339     ZEBRA_RES res = ZEBRA_OK;
2340     unsigned reg_id;
2341     char *search_type = NULL;
2342     char rank_type[128];
2343     int complete_flag;
2344     int sort_flag;
2345     char termz[IT_MAX_WORD+1];
2346     int xpath_len;
2347     int xpath_use = 0;
2348     struct xpath_location_step xpath[10];
2349
2350     if (!log_level_set)
2351     {
2352         log_level_rpn = yaz_log_module_level("rpn");
2353         log_level_set = 1;
2354     }
2355     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2356                     rank_type, &complete_flag, &sort_flag);
2357
2358     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2359     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2360     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2361     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2362
2363     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2364         return ZEBRA_FAIL;
2365
2366     if (sort_flag)
2367         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2368                              rank_type, rset_nmem, rset, kc);
2369     /* consider if an X-Path query is used */
2370     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2371     if (xpath_len >= 0)
2372     {
2373         xpath_use = 1016;  /* searching for element by default */
2374         if (xpath[xpath_len-1].part[0] == '@')
2375             xpath_use = 1015;  /* last step an attribute .. */
2376     }
2377
2378     /* search using one of the various search type strategies
2379        termz is our UTF-8 search term
2380        attributeSet is top-level default attribute set
2381        stream is ODR for search
2382        reg_id is the register type
2383        complete_flag is 1 for complete subfield, 0 for incomplete
2384        xpath_use is use-attribute to be used for X-Path search, 0 for none
2385     */
2386     if (!strcmp(search_type, "phrase"))
2387     {
2388         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2389                                     reg_id, complete_flag, rank_type,
2390                                     xpath_use,
2391                                     num_bases, basenames, rset_nmem,
2392                                     rset, kc);
2393     }
2394     else if (!strcmp(search_type, "and-list"))
2395     {
2396         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2397                                       reg_id, complete_flag, rank_type,
2398                                       xpath_use,
2399                                       num_bases, basenames, rset_nmem,
2400                                       rset, kc);
2401     }
2402     else if (!strcmp(search_type, "or-list"))
2403     {
2404         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2405                                      reg_id, complete_flag, rank_type,
2406                                      xpath_use,
2407                                      num_bases, basenames, rset_nmem,
2408                                      rset, kc);
2409     }
2410     else if (!strcmp(search_type, "local"))
2411     {
2412         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2413                                    rank_type, rset_nmem, rset, kc);
2414     }
2415     else if (!strcmp(search_type, "numeric"))
2416     {
2417         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2418                                      reg_id, complete_flag, rank_type,
2419                                      xpath_use,
2420                                      num_bases, basenames, rset_nmem,
2421                                      rset, kc);
2422     }
2423     else
2424     {
2425         zebra_setError(zh, YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE, 0);
2426         res = ZEBRA_FAIL;
2427     }
2428     if (res != ZEBRA_OK)
2429         return res;
2430     if (!*rset)
2431         return ZEBRA_FAIL;
2432     return rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2433                             stream, rank_type, *rset,
2434                             xpath_len, xpath, rset_nmem, rset, kc);
2435 }
2436
2437 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2438                                       oid_value attributeSet,
2439                                       NMEM stream, NMEM rset_nmem,
2440                                       Z_SortKeySpecList *sort_sequence,
2441                                       int num_bases, char **basenames,
2442                                       RSET **result_sets, int *num_result_sets,
2443                                       Z_Operator *parent_op,
2444                                       struct rset_key_control *kc);
2445
2446 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2447                          oid_value attributeSet,
2448                          NMEM stream, NMEM rset_nmem,
2449                          Z_SortKeySpecList *sort_sequence,
2450                          int num_bases, char **basenames,
2451                          RSET *result_set)
2452 {
2453     RSET *result_sets = 0;
2454     int num_result_sets = 0;
2455     ZEBRA_RES res;
2456     struct rset_key_control *kc = zebra_key_control_create(zh);
2457
2458     res = rpn_search_structure(zh, zs, attributeSet,
2459                                stream, rset_nmem,
2460                                sort_sequence,
2461                                num_bases, basenames,
2462                                &result_sets, &num_result_sets,
2463                                0 /* no parent op */,
2464                                kc);
2465     if (res != ZEBRA_OK)
2466     {
2467         int i;
2468         for (i = 0; i<num_result_sets; i++)
2469             rset_delete(result_sets[i]);
2470         *result_set = 0;
2471     }
2472     else
2473     {
2474         assert(num_result_sets == 1);
2475         assert(result_sets);
2476         assert(*result_sets);
2477         *result_set = *result_sets;
2478     }
2479     (*kc->dec)(kc);
2480     return res;
2481 }
2482
2483 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2484                                oid_value attributeSet,
2485                                NMEM stream, NMEM rset_nmem,
2486                                Z_SortKeySpecList *sort_sequence,
2487                                int num_bases, char **basenames,
2488                                RSET **result_sets, int *num_result_sets,
2489                                Z_Operator *parent_op,
2490                                struct rset_key_control *kc)
2491 {
2492     *num_result_sets = 0;
2493     if (zs->which == Z_RPNStructure_complex)
2494     {
2495         ZEBRA_RES res;
2496         Z_Operator *zop = zs->u.complex->roperator;
2497         RSET *result_sets_l = 0;
2498         int num_result_sets_l = 0;
2499         RSET *result_sets_r = 0;
2500         int num_result_sets_r = 0;
2501
2502         res = rpn_search_structure(zh, zs->u.complex->s1,
2503                                    attributeSet, stream, rset_nmem,
2504                                    sort_sequence,
2505                                    num_bases, basenames,
2506                                    &result_sets_l, &num_result_sets_l,
2507                                    zop, kc);
2508         if (res != ZEBRA_OK)
2509         {
2510             int i;
2511             for (i = 0; i<num_result_sets_l; i++)
2512                 rset_delete(result_sets_l[i]);
2513             return res;
2514         }
2515         res = rpn_search_structure(zh, zs->u.complex->s2,
2516                                    attributeSet, stream, rset_nmem,
2517                                    sort_sequence,
2518                                    num_bases, basenames,
2519                                    &result_sets_r, &num_result_sets_r,
2520                                    zop, kc);
2521         if (res != ZEBRA_OK)
2522         {
2523             int i;
2524             for (i = 0; i<num_result_sets_l; i++)
2525                 rset_delete(result_sets_l[i]);
2526             for (i = 0; i<num_result_sets_r; i++)
2527                 rset_delete(result_sets_r[i]);
2528             return res;
2529         }
2530
2531         /* make a new list of result for all children */
2532         *num_result_sets = num_result_sets_l + num_result_sets_r;
2533         *result_sets = nmem_malloc(stream, *num_result_sets *
2534                                    sizeof(**result_sets));
2535         memcpy(*result_sets, result_sets_l,
2536                num_result_sets_l * sizeof(**result_sets));
2537         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2538                num_result_sets_r * sizeof(**result_sets));
2539
2540         if (!parent_op || parent_op->which != zop->which
2541             || (zop->which != Z_Operator_and &&
2542                 zop->which != Z_Operator_or))
2543         {
2544             /* parent node different from this one (or non-present) */
2545             /* we must combine result sets now */
2546             RSET rset;
2547             switch (zop->which)
2548             {
2549             case Z_Operator_and:
2550                 rset = rsmulti_and_create(rset_nmem, kc,
2551                                           kc->scope,
2552                                           *num_result_sets, *result_sets);
2553                 break;
2554             case Z_Operator_or:
2555                 rset = rsmulti_or_create(rset_nmem, kc,
2556                                          kc->scope, 0, /* termid */
2557                                          *num_result_sets, *result_sets);
2558                 break;
2559             case Z_Operator_and_not:
2560                 rset = rsbool_create_not(rset_nmem, kc,
2561                                          kc->scope,
2562                                          (*result_sets)[0],
2563                                          (*result_sets)[1]);
2564                 break;
2565             case Z_Operator_prox:
2566                 if (zop->u.prox->which != Z_ProximityOperator_known)
2567                 {
2568                     zebra_setError(zh,
2569                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2570                                    0);
2571                     return ZEBRA_FAIL;
2572                 }
2573                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2574                 {
2575                     zebra_setError_zint(zh,
2576                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2577                                         *zop->u.prox->u.known);
2578                     return ZEBRA_FAIL;
2579                 }
2580                 else
2581                 {
2582                     rset = rsprox_create(rset_nmem, kc,
2583                                          kc->scope,
2584                                          *num_result_sets, *result_sets,
2585                                          *zop->u.prox->ordered,
2586                                          (!zop->u.prox->exclusion ?
2587                                           0 : *zop->u.prox->exclusion),
2588                                          *zop->u.prox->relationType,
2589                                          *zop->u.prox->distance );
2590                 }
2591                 break;
2592             default:
2593                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2594                 return ZEBRA_FAIL;
2595             }
2596             *num_result_sets = 1;
2597             *result_sets = nmem_malloc(stream, *num_result_sets *
2598                                        sizeof(**result_sets));
2599             (*result_sets)[0] = rset;
2600         }
2601     }
2602     else if (zs->which == Z_RPNStructure_simple)
2603     {
2604         RSET rset;
2605         ZEBRA_RES res;
2606
2607         if (zs->u.simple->which == Z_Operand_APT)
2608         {
2609             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2610             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2611                                  attributeSet, stream, sort_sequence,
2612                                  num_bases, basenames, rset_nmem, &rset,
2613                                  kc);
2614             if (res != ZEBRA_OK)
2615                 return res;
2616         }
2617         else if (zs->u.simple->which == Z_Operand_resultSetId)
2618         {
2619             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2620             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2621             if (!rset)
2622             {
2623                 zebra_setError(zh,
2624                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2625                                zs->u.simple->u.resultSetId);
2626                 return ZEBRA_FAIL;
2627             }
2628             rset_dup(rset);
2629         }
2630         else
2631         {
2632             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2633             return ZEBRA_FAIL;
2634         }
2635         *num_result_sets = 1;
2636         *result_sets = nmem_malloc(stream, *num_result_sets *
2637                                    sizeof(**result_sets));
2638         (*result_sets)[0] = rset;
2639     }
2640     else
2641     {
2642         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2643         return ZEBRA_FAIL;
2644     }
2645     return ZEBRA_OK;
2646 }
2647
2648 struct scan_info_entry {
2649     char *term;
2650     ISAM_P isam_p;
2651 };
2652
2653 struct scan_info {
2654     struct scan_info_entry *list;
2655     ODR odr;
2656     int before, after;
2657     char prefix[20];
2658 };
2659
2660 static int scan_handle (char *name, const char *info, int pos, void *client)
2661 {
2662     int len_prefix, idx;
2663     struct scan_info *scan_info = (struct scan_info *) client;
2664
2665     len_prefix = strlen(scan_info->prefix);
2666     if (memcmp (name, scan_info->prefix, len_prefix))
2667         return 1;
2668     if (pos > 0)
2669         idx = scan_info->after - pos + scan_info->before;
2670     else
2671         idx = - pos - 1;
2672
2673     if (idx < 0)
2674         return 0;
2675     scan_info->list[idx].term = (char *)
2676         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2677     strcpy(scan_info->list[idx].term, name + len_prefix);
2678     assert (*info == sizeof(ISAM_P));
2679     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2680     return 0;
2681 }
2682
2683 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2684                               char **dst, const char *src)
2685 {
2686     char term_src[IT_MAX_WORD];
2687     char term_dst[IT_MAX_WORD];
2688
2689     zebra_term_untrans (zh, reg_type, term_src, src);
2690
2691     if (zh->iconv_from_utf8 != 0)
2692     {
2693         int len;
2694         char *inbuf = term_src;
2695         size_t inleft = strlen(term_src);
2696         char *outbuf = term_dst;
2697         size_t outleft = sizeof(term_dst)-1;
2698         size_t ret;
2699
2700         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2701                          &outbuf, &outleft);
2702         if (ret == (size_t)(-1))
2703             len = 0;
2704         else
2705             len = outbuf - term_dst;
2706         *dst = nmem_malloc(stream, len + 1);
2707         if (len > 0)
2708             memcpy (*dst, term_dst, len);
2709         (*dst)[len] = '\0';
2710     }
2711     else
2712         *dst = nmem_strdup(stream, term_src);
2713 }
2714
2715 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2716 {
2717     zint psysno = 0;
2718     struct it_key key;
2719     RSFD rfd;
2720
2721     yaz_log(YLOG_DEBUG, "count_set");
2722
2723     rset->hits_limit = zh->approx_limit;
2724
2725     *count = 0;
2726     rfd = rset_open(rset, RSETF_READ);
2727     while (rset_read(rfd, &key,0 /* never mind terms */))
2728     {
2729         if (key.mem[0] != psysno)
2730         {
2731             psysno = key.mem[0];
2732             if (rfd->counted_items >= rset->hits_limit)
2733                 break;
2734         }
2735     }
2736     rset_close (rfd);
2737     *count = rset->hits_count;
2738 }
2739
2740 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2741                    oid_value attributeset,
2742                    int num_bases, char **basenames,
2743                    int *position, int *num_entries, ZebraScanEntry **list,
2744                    int *is_partial, RSET limit_set, int return_zero)
2745 {
2746     int i;
2747     int pos = *position;
2748     int num = *num_entries;
2749     int before;
2750     int after;
2751     int base_no;
2752     char termz[IT_MAX_WORD+20];
2753     AttrType use;
2754     int use_value;
2755     const char *use_string = 0;
2756     struct scan_info *scan_info_array;
2757     ZebraScanEntry *glist;
2758     int ords[32], ord_no = 0;
2759     int ptr[32];
2760
2761     int bases_ok = 0;     /* no of databases with OK attribute */
2762     int errCode = 0;      /* err code (if any is not OK) */
2763     char *errString = 0;  /* addinfo */
2764
2765     unsigned reg_id;
2766     char *search_type = NULL;
2767     char rank_type[128];
2768     int complete_flag;
2769     int sort_flag;
2770     NMEM rset_nmem = NULL;
2771     struct rset_key_control *kc = 0;
2772
2773     *list = 0;
2774     *is_partial = 0;
2775
2776     if (attributeset == VAL_NONE)
2777         attributeset = VAL_BIB1;
2778
2779     if (!limit_set)
2780     {
2781         AttrType termset;
2782         int termset_value_numeric;
2783         const char *termset_value_string;
2784         attr_init(&termset, zapt, 8);
2785         termset_value_numeric =
2786             attr_find_ex(&termset, NULL, &termset_value_string);
2787         if (termset_value_numeric != -1)
2788         {
2789             char resname[32];
2790             const char *termset_name = 0;
2791
2792             if (termset_value_numeric != -2)
2793             {
2794
2795                 sprintf(resname, "%d", termset_value_numeric);
2796                 termset_name = resname;
2797             }
2798             else
2799                 termset_name = termset_value_string;
2800
2801             limit_set = resultSetRef (zh, termset_name);
2802         }
2803     }
2804
2805     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2806             pos, num, attributeset);
2807
2808     attr_init(&use, zapt, 1);
2809     use_value = attr_find_ex(&use, &attributeset, &use_string);
2810
2811     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2812                         rank_type, &complete_flag, &sort_flag))
2813     {
2814         *num_entries = 0;
2815         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2816         return ZEBRA_FAIL;
2817     }
2818     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2819
2820     if (use_value == -1)
2821         use_value = 1016;
2822     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2823     {
2824         data1_local_attribute *local_attr;
2825         attent attp;
2826         int ord;
2827
2828         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2829         {
2830             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2831                            basenames[base_no]);
2832             *num_entries = 0;
2833             return ZEBRA_FAIL;
2834         }
2835
2836         if (use_string &&
2837             (ord = zebraExplain_lookup_attr_str(zh->reg->zei, reg_id,
2838                                                 use_string)) >= 0)
2839         {
2840             /* we have a match for a raw string attribute */
2841             if (ord > 0)
2842                 ords[ord_no++] = ord;
2843             attp.local_attributes = 0;  /* no more attributes */
2844         }
2845         else
2846         {
2847             int r;
2848
2849             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2850                                       use_string)))
2851             {
2852                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2853                         attributeset, use_value);
2854                 if (r == -1)
2855                 {
2856                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2857                     if (use_string)
2858                         zebra_setError(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
2859                                        use_string);
2860                     else
2861                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
2862                                             use_value);
2863                 }
2864                 else
2865                 {
2866                     zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_SET, 0);
2867                 }
2868                 continue;
2869             }
2870         }
2871         bases_ok++;
2872         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2873              local_attr = local_attr->next)
2874         {
2875             ord = zebraExplain_lookup_attr_su(zh->reg->zei, reg_id,
2876                                               attp.attset_ordinal,
2877                                               local_attr->local);
2878             if (ord > 0)
2879                 ords[ord_no++] = ord;
2880         }
2881     }
2882     if (!bases_ok && errCode)
2883     {
2884         zebra_setError(zh, errCode, errString);
2885         *num_entries = 0;
2886         return ZEBRA_FAIL;
2887     }
2888     if (ord_no == 0)
2889     {
2890         *num_entries = 0;
2891         return ZEBRA_OK;
2892     }
2893     /* prepare dictionary scanning */
2894     if (num < 1)
2895     {
2896         *num_entries = 0;
2897         return ZEBRA_OK;
2898     }
2899     before = pos-1;
2900     if (before < 0)
2901         before = 0;
2902     after = 1+num-pos;
2903     if (after < 0)
2904         after = 0;
2905     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2906             "after=%d before+after=%d",
2907             pos, num, before, after, before+after);
2908     scan_info_array = (struct scan_info *)
2909         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2910     for (i = 0; i < ord_no; i++)
2911     {
2912         int j, prefix_len = 0;
2913         int before_tmp = before, after_tmp = after;
2914         struct scan_info *scan_info = scan_info_array + i;
2915         struct rpn_char_map_info rcmi;
2916
2917         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2918
2919         scan_info->before = before;
2920         scan_info->after = after;
2921         scan_info->odr = stream;
2922
2923         scan_info->list = (struct scan_info_entry *)
2924             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2925         for (j = 0; j<before+after; j++)
2926             scan_info->list[j].term = NULL;
2927
2928         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2929 #if REG_TYPE_PREFIX
2930         termz[prefix_len++] = reg_id;
2931 #endif
2932         termz[prefix_len] = 0;
2933         strcpy(scan_info->prefix, termz);
2934
2935         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2936             return ZEBRA_FAIL;
2937
2938         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2939                   scan_info, scan_handle);
2940     }
2941     glist = (ZebraScanEntry *)
2942         odr_malloc(stream, (before+after)*sizeof(*glist));
2943
2944     rset_nmem = nmem_create();
2945     kc = zebra_key_control_create(zh);
2946
2947     /* consider terms after main term */
2948     for (i = 0; i < ord_no; i++)
2949         ptr[i] = before;
2950
2951     *is_partial = 0;
2952     for (i = 0; i<after; i++)
2953     {
2954         int j, j0 = -1;
2955         const char *mterm = NULL;
2956         const char *tst;
2957         RSET rset = 0;
2958         int lo = i + pos-1; /* offset in result list */
2959
2960         /* find: j0 is the first of the minimal values */
2961         for (j = 0; j < ord_no; j++)
2962         {
2963             if (ptr[j] < before+after && ptr[j] >= 0 &&
2964                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2965                 (!mterm || strcmp (tst, mterm) < 0))
2966             {
2967                 j0 = j;
2968                 mterm = tst;
2969             }
2970         }
2971         if (j0 == -1)
2972             break;  /* no value found, stop */
2973
2974         /* get result set for first one , but only if it's within bounds */
2975         if (lo >= 0)
2976         {
2977             /* get result set for first term */
2978             zebra_term_untrans_iconv(zh, stream->mem, reg_id,
2979                                      &glist[lo].term, mterm);
2980             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2981                               glist[lo].term, strlen(glist[lo].term),
2982                               NULL, 0, zapt->term->which, rset_nmem,
2983                               kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
2984                               0 /* term_ref_id_str */);
2985         }
2986         ptr[j0]++; /* move index for this set .. */
2987         /* get result set for remaining scan terms */
2988         for (j = j0+1; j<ord_no; j++)
2989         {
2990             if (ptr[j] < before+after && ptr[j] >= 0 &&
2991                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2992                 !strcmp (tst, mterm))
2993             {
2994                 if (lo >= 0)
2995                 {
2996                     RSET rsets[2];
2997
2998                     rsets[0] = rset;
2999                     rsets[1] =
3000                         rset_trunc(
3001                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
3002                             glist[lo].term,
3003                             strlen(glist[lo].term), NULL, 0,
3004                             zapt->term->which,rset_nmem,
3005                             kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3006                             0 /* term_ref_id_str */ );
3007                     rset = rsmulti_or_create(rset_nmem, kc,
3008                                              kc->scope, 0 /* termid */,
3009                                              2, rsets);
3010                 }
3011                 ptr[j]++;
3012             }
3013         }
3014         if (lo >= 0)
3015         {
3016             zint count;
3017             /* merge with limit_set if given */
3018             if (limit_set)
3019             {
3020                 RSET rsets[2];
3021                 rsets[0] = rset;
3022                 rsets[1] = rset_dup(limit_set);
3023
3024                 rset = rsmulti_and_create(rset_nmem, kc,
3025                                           kc->scope,
3026                                           2, rsets);
3027             }
3028             /* count it */
3029             count_set(zh, rset, &count);
3030             glist[lo].occurrences = count;
3031             rset_delete(rset);
3032         }
3033     }
3034     if (i < after)
3035     {
3036         *num_entries -= (after-i);
3037         *is_partial = 1;
3038         if (*num_entries < 0)
3039         {
3040             (*kc->dec)(kc);
3041             nmem_destroy(rset_nmem);
3042             *num_entries = 0;
3043             return ZEBRA_OK;
3044         }
3045     }
3046     /* consider terms before main term */
3047     for (i = 0; i<ord_no; i++)
3048         ptr[i] = 0;
3049
3050     for (i = 0; i<before; i++)
3051     {
3052         int j, j0 = -1;
3053         const char *mterm = NULL;
3054         const char *tst;
3055         RSET rset;
3056         int lo = before-1-i; /* offset in result list */
3057         zint count;
3058
3059         for (j = 0; j <ord_no; j++)
3060         {
3061             if (ptr[j] < before && ptr[j] >= 0 &&
3062                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3063                 (!mterm || strcmp (tst, mterm) > 0))
3064             {
3065                 j0 = j;
3066                     mterm = tst;
3067             }
3068         }
3069         if (j0 == -1)
3070             break;
3071
3072         zebra_term_untrans_iconv(zh, stream->mem, reg_id,
3073                                  &glist[lo].term, mterm);
3074
3075         rset = rset_trunc
3076             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
3077              glist[lo].term, strlen(glist[lo].term),
3078              NULL, 0, zapt->term->which, rset_nmem,
3079              kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3080              0 /* term_ref_id_str */);
3081
3082         ptr[j0]++;
3083
3084         for (j = j0+1; j<ord_no; j++)
3085         {
3086             if (ptr[j] < before && ptr[j] >= 0 &&
3087                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3088                 !strcmp (tst, mterm))
3089             {
3090                 RSET rsets[2];
3091
3092                 rsets[0] = rset;
3093                 rsets[1] = rset_trunc(
3094                     zh,
3095                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
3096                     glist[lo].term,
3097                     strlen(glist[lo].term), NULL, 0,
3098                     zapt->term->which, rset_nmem,
3099                     kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3100                     0 /* term_ref_id_str */);
3101                 rset = rsmulti_or_create(rset_nmem, kc,
3102                                          kc->scope, 0 /* termid */, 2, rsets);
3103
3104                 ptr[j]++;
3105             }
3106         }
3107         if (limit_set)
3108         {
3109             RSET rsets[2];
3110             rsets[0] = rset;
3111             rsets[1] = rset_dup(limit_set);
3112
3113             rset = rsmulti_and_create(rset_nmem, kc,
3114                                       kc->scope, 2, rsets);
3115         }
3116         count_set(zh, rset, &count);
3117         glist[lo].occurrences = count;
3118         rset_delete (rset);
3119     }
3120     (*kc->dec)(kc);
3121     nmem_destroy(rset_nmem);
3122     i = before-i;
3123     if (i)
3124     {
3125         *is_partial = 1;
3126         *position -= i;
3127         *num_entries -= i;
3128         if (*num_entries <= 0)
3129         {
3130             *num_entries = 0;
3131             return ZEBRA_OK;
3132         }
3133     }
3134
3135     *list = glist + i;               /* list is set to first 'real' entry */
3136
3137     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3138             *position, *num_entries);
3139     return ZEBRA_OK;
3140 }
3141