index/zrpn.c

   1 /* $Id: zrpn.c,v 1.208 2005-11-29 11:38:34 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 typedef struct
  47 {
  48     int type;
  49     int major;
  50     int minor;
  51     Z_AttributesPlusTerm *zapt;
  52 } AttrType;
  53
  54 static int log_level_set = 0;
  55 static int log_level_rpn = 0;
  56
  57 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  58 {
  59     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  60     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  61 #if 0
  62     if (out && *out)
  63     {
  64         const char *outp = *out;
  65         yaz_log(YLOG_LOG, "---");
  66         while (*outp)
  67         {
  68             yaz_log(YLOG_LOG, "%02X", *outp);
  69             outp++;
  70         }
  71     }
  72 #endif
  73     return out;
  74 }
  75
  76 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  77                                   struct rpn_char_map_info *map_info)
  78 {
  79     map_info->zm = reg->zebra_maps;
  80     map_info->reg_type = reg_type;
  81     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  82 }
  83
  84 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  85                         const char **string_value)
  86 {
  87     int num_attributes;
  88
  89     num_attributes = src->zapt->attributes->num_attributes;
  90     while (src->major < num_attributes)
  91     {
  92         Z_AttributeElement *element;
  93
  94         element = src->zapt->attributes->attributes[src->major];
  95         if (src->type == *element->attributeType)
  96         {
  97             switch (element->which)
  98             {
  99             case Z_AttributeValue_numeric:
 100                 ++(src->major);
 101                 if (element->attributeSet && attributeSetP)
 102                 {
 103                     oident *attrset;
 104
 105                     attrset = oid_getentbyoid(element->attributeSet);
 106                     *attributeSetP = attrset->value;
 107                 }
 108                 return *element->value.numeric;
 109                 break;
 110             case Z_AttributeValue_complex:
 111                 if (src->minor >= element->value.complex->num_list)
 112                     break;
 113                 if (element->attributeSet && attributeSetP)
 114                 {
 115                     oident *attrset;
 116
 117                     attrset = oid_getentbyoid(element->attributeSet);
 118                     *attributeSetP = attrset->value;
 119                 }
 120                 if (element->value.complex->list[src->minor]->which ==
 121                     Z_StringOrNumeric_numeric)
 122                 {
 123                     ++(src->minor);
 124                     return
 125                         *element->value.complex->list[src->minor-1]->u.numeric;
 126                 }
 127                 else if (element->value.complex->list[src->minor]->which ==
 128                          Z_StringOrNumeric_string)
 129                 {
 130                     if (!string_value)
 131                         break;
 132                     ++(src->minor);
 133                     *string_value =
 134                         element->value.complex->list[src->minor-1]->u.string;
 135                     return -2;
 136                 }
 137                 else
 138                     break;
 139             default:
 140                 assert(0);
 141             }
 142         }
 143         ++(src->major);
 144     }
 145     return -1;
 146 }
 147
 148 static int attr_find(AttrType *src, oid_value *attributeSetP)
 149 {
 150     return attr_find_ex(src, attributeSetP, 0);
 151 }
 152
 153 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 154                        int type)
 155 {
 156     src->zapt = zapt;
 157     src->type = type;
 158     src->major = 0;
 159     src->minor = 0;
 160 }
 161
 162 #define TERM_COUNT
 163
 164 struct grep_info {
 165 #ifdef TERM_COUNT
 166     int *term_no;
 167 #endif
 168     ISAM_P *isam_p_buf;
 169     int isam_p_size;
 170     int isam_p_indx;
 171     ZebraHandle zh;
 172     int reg_type;
 173     ZebraSet termset;
 174 };
 175
 176 void zebra_term_untrans(ZebraHandle zh, int reg_type,
 177                         char *dst, const char *src)
 178 {
 179     int len = 0;
 180     while (*src)
 181     {
 182         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 183                                            reg_type, &src);
 184         if (!cp && len < IT_MAX_WORD-1)
 185             dst[len++] = *src++;
 186         else
 187             while (*cp && len < IT_MAX_WORD-1)
 188                 dst[len++] = *cp++;
 189     }
 190     dst[len] = '\0';
 191 }
 192
 193 static void add_isam_p(const char *name, const char *info,
 194                        struct grep_info *p)
 195 {
 196     if (!log_level_set)
 197     {
 198         log_level_rpn = yaz_log_module_level("rpn");
 199         log_level_set = 1;
 200     }
 201     if (p->isam_p_indx == p->isam_p_size)
 202     {
 203         ISAM_P *new_isam_p_buf;
 204 #ifdef TERM_COUNT
 205         int *new_term_no;
 206 #endif
 207         p->isam_p_size = 2*p->isam_p_size + 100;
 208         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 209                                             p->isam_p_size);
 210         if (p->isam_p_buf)
 211         {
 212             memcpy(new_isam_p_buf, p->isam_p_buf,
 213                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 214             xfree(p->isam_p_buf);
 215         }
 216         p->isam_p_buf = new_isam_p_buf;
 217
 218 #ifdef TERM_COUNT
 219         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 220         if (p->term_no)
 221         {
 222             memcpy(new_term_no, p->isam_p_buf,
 223                     p->isam_p_indx * sizeof(*p->term_no));
 224             xfree(p->term_no);
 225         }
 226         p->term_no = new_term_no;
 227 #endif
 228     }
 229     assert(*info == sizeof(*p->isam_p_buf));
 230     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 231
 232 #if 1
 233     if (p->termset)
 234     {
 235         const char *db;
 236         int set, use;
 237         char term_tmp[IT_MAX_WORD];
 238         int ord = 0;
 239         int len = key_SU_decode (&ord, (const unsigned char *) name);
 240
 241         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 242         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 243         zebraExplain_lookup_ord (p->zh->reg->zei,
 244                                  ord, 0 /* index_type */, &db, &set, &use);
 245         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 246
 247         resultSetAddTerm(p->zh, p->termset, name[len], db,
 248                          set, use, term_tmp);
 249     }
 250 #endif
 251     (p->isam_p_indx)++;
 252 }
 253
 254 static int grep_handle(char *name, const char *info, void *p)
 255 {
 256     add_isam_p(name, info, (struct grep_info *) p);
 257     return 0;
 258 }
 259
 260 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 261                     const char *ct1, const char *ct2, int first)
 262 {
 263     const char *s1, *s0 = *src;
 264     const char **map;
 265
 266     /* skip white space */
 267     while (*s0)
 268     {
 269         if (ct1 && strchr(ct1, *s0))
 270             break;
 271         if (ct2 && strchr(ct2, *s0))
 272             break;
 273         s1 = s0;
 274         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 275         if (**map != *CHR_SPACE)
 276             break;
 277         s0 = s1;
 278     }
 279     *src = s0;
 280     return *s0;
 281 }
 282
 283
 284 static void esc_str(char *out_buf, int out_size,
 285                     const char *in_buf, int in_size)
 286 {
 287     int k;
 288
 289     assert(out_buf);
 290     assert(in_buf);
 291     assert(out_size > 20);
 292     *out_buf = '\0';
 293     for (k = 0; k<in_size; k++)
 294     {
 295         int c = in_buf[k] & 0xff;
 296         int pc;
 297         if (c < 32 || c > 126)
 298             pc = '?';
 299         else
 300             pc = c;
 301         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 302         if (strlen(out_buf) > out_size-20)
 303         {
 304             strcat(out_buf, "..");
 305             break;
 306         }
 307     }
 308 }
 309
 310 #define REGEX_CHARS " []()|.*+?!"
 311
 312 /* term_100: handle term, where trunc = none(no operators at all) */
 313 static int term_100(ZebraMaps zebra_maps, int reg_type,
 314                     const char **src, char *dst, int space_split,
 315                     char *dst_term)
 316 {
 317     const char *s0;
 318     const char **map;
 319     int i = 0;
 320     int j = 0;
 321
 322     const char *space_start = 0;
 323     const char *space_end = 0;
 324
 325     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 326         return 0;
 327     s0 = *src;
 328     while (*s0)
 329     {
 330         const char *s1 = s0;
 331         int q_map_match = 0;
 332         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 333                                 &q_map_match);
 334         if (space_split)
 335         {
 336             if (**map == *CHR_SPACE)
 337                 break;
 338         }
 339         else  /* complete subfield only. */
 340         {
 341             if (**map == *CHR_SPACE)
 342             {   /* save space mapping for later  .. */
 343                 space_start = s1;
 344                 space_end = s0;
 345                 continue;
 346             }
 347             else if (space_start)
 348             {   /* reload last space */
 349                 while (space_start < space_end)
 350                 {
 351                     if (strchr(REGEX_CHARS, *space_start))
 352                         dst[i++] = '\\';
 353                     dst_term[j++] = *space_start;
 354                     dst[i++] = *space_start++;
 355                 }
 356                 /* and reset */
 357                 space_start = space_end = 0;
 358             }
 359         }
 360         /* add non-space char */
 361         memcpy(dst_term+j, s1, s0 - s1);
 362         j += (s0 - s1);
 363         if (!q_map_match)
 364         {
 365             while (s1 < s0)
 366             {
 367                 if (strchr(REGEX_CHARS, *s1))
 368                     dst[i++] = '\\';
 369                 dst[i++] = *s1++;
 370             }
 371         }
 372         else
 373         {
 374             char tmpbuf[80];
 375             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 376
 377             strcpy(dst + i, map[0]);
 378             i += strlen(map[0]);
 379         }
 380     }
 381     dst[i] = '\0';
 382     dst_term[j] = '\0';
 383     *src = s0;
 384     return i;
 385 }
 386
 387 /* term_101: handle term, where trunc = Process # */
 388 static int term_101(ZebraMaps zebra_maps, int reg_type,
 389                     const char **src, char *dst, int space_split,
 390                     char *dst_term)
 391 {
 392     const char *s0;
 393     const char **map;
 394     int i = 0;
 395     int j = 0;
 396
 397     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 398         return 0;
 399     s0 = *src;
 400     while (*s0)
 401     {
 402         if (*s0 == '#')
 403         {
 404             dst[i++] = '.';
 405             dst[i++] = '*';
 406             dst_term[j++] = *s0++;
 407         }
 408         else
 409         {
 410             const char *s1 = s0;
 411             int q_map_match = 0;
 412             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 413                                     &q_map_match);
 414             if (space_split && **map == *CHR_SPACE)
 415                 break;
 416
 417             /* add non-space char */
 418             memcpy(dst_term+j, s1, s0 - s1);
 419             j += (s0 - s1);
 420             if (!q_map_match)
 421             {
 422                 while (s1 < s0)
 423                 {
 424                     if (strchr(REGEX_CHARS, *s1))
 425                         dst[i++] = '\\';
 426                     dst[i++] = *s1++;
 427                 }
 428             }
 429             else
 430             {
 431                 char tmpbuf[80];
 432                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 433
 434                 strcpy(dst + i, map[0]);
 435                 i += strlen(map[0]);
 436             }
 437         }
 438     }
 439     dst[i] = '\0';
 440     dst_term[j++] = '\0';
 441     *src = s0;
 442     return i;
 443 }
 444
 445 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 446 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 447                     char *dst, int *errors, int space_split,
 448                     char *dst_term)
 449 {
 450     int i = 0;
 451     int j = 0;
 452     const char *s0;
 453     const char **map;
 454
 455     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 456         return 0;
 457     s0 = *src;
 458     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 459         isdigit(((const unsigned char *)s0)[1]))
 460     {
 461         *errors = s0[1] - '0';
 462         s0 += 3;
 463         if (*errors > 3)
 464             *errors = 3;
 465     }
 466     while (*s0)
 467     {
 468         if (strchr("^\\()[].*+?|-", *s0))
 469         {
 470             dst_term[j++] = *s0;
 471             dst[i++] = *s0++;
 472         }
 473         else
 474         {
 475             const char *s1 = s0;
 476             int q_map_match = 0;
 477             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 478                                     &q_map_match);
 479             if (space_split && **map == *CHR_SPACE)
 480                 break;
 481
 482             /* add non-space char */
 483             memcpy(dst_term+j, s1, s0 - s1);
 484             j += (s0 - s1);
 485             if (!q_map_match)
 486             {
 487                 while (s1 < s0)
 488                 {
 489                     if (strchr(REGEX_CHARS, *s1))
 490                         dst[i++] = '\\';
 491                     dst[i++] = *s1++;
 492                 }
 493             }
 494             else
 495             {
 496                 char tmpbuf[80];
 497                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 498
 499                 strcpy(dst + i, map[0]);
 500                 i += strlen(map[0]);
 501             }
 502         }
 503     }
 504     dst[i] = '\0';
 505     dst_term[j] = '\0';
 506     *src = s0;
 507
 508     return i;
 509 }
 510
 511 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 512 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 513                     char *dst, int space_split, char *dst_term)
 514 {
 515     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 516                     dst_term);
 517 }
 518
 519
 520 /* term_104: handle term, where trunc = Process # and ! */
 521 static int term_104(ZebraMaps zebra_maps, int reg_type,
 522                     const char **src, char *dst, int space_split,
 523                     char *dst_term)
 524 {
 525     const char *s0;
 526     const char **map;
 527     int i = 0;
 528     int j = 0;
 529
 530     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 531         return 0;
 532     s0 = *src;
 533     while (*s0)
 534     {
 535         if (*s0 == '?')
 536         {
 537             dst_term[j++] = *s0++;
 538             if (*s0 >= '0' && *s0 <= '9')
 539             {
 540                 int limit = 0;
 541                 while (*s0 >= '0' && *s0 <= '9')
 542                 {
 543                     limit = limit * 10 + (*s0 - '0');
 544                     dst_term[j++] = *s0++;
 545                 }
 546                 if (limit > 20)
 547                     limit = 20;
 548                 while (--limit >= 0)
 549                 {
 550                     dst[i++] = '.';
 551                     dst[i++] = '?';
 552                 }
 553             }
 554             else
 555             {
 556                 dst[i++] = '.';
 557                 dst[i++] = '*';
 558             }
 559         }
 560         else if (*s0 == '*')
 561         {
 562             dst[i++] = '.';
 563             dst[i++] = '*';
 564             dst_term[j++] = *s0++;
 565         }
 566         else if (*s0 == '#')
 567         {
 568             dst[i++] = '.';
 569             dst_term[j++] = *s0++;
 570         }
 571         else
 572         {
 573             const char *s1 = s0;
 574             int q_map_match = 0;
 575             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 576                                     &q_map_match);
 577             if (space_split && **map == *CHR_SPACE)
 578                 break;
 579
 580             /* add non-space char */
 581             memcpy(dst_term+j, s1, s0 - s1);
 582             j += (s0 - s1);
 583             if (!q_map_match)
 584             {
 585                 while (s1 < s0)
 586                 {
 587                     if (strchr(REGEX_CHARS, *s1))
 588                         dst[i++] = '\\';
 589                     dst[i++] = *s1++;
 590                 }
 591             }
 592             else
 593             {
 594                 char tmpbuf[80];
 595                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 596
 597                 strcpy(dst + i, map[0]);
 598                 i += strlen(map[0]);
 599             }
 600         }
 601     }
 602     dst[i] = '\0';
 603     dst_term[j++] = '\0';
 604     *src = s0;
 605     return i;
 606 }
 607
 608 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 609 static int term_105(ZebraMaps zebra_maps, int reg_type,
 610                     const char **src, char *dst, int space_split,
 611                     char *dst_term, int right_truncate)
 612 {
 613     const char *s0;
 614     const char **map;
 615     int i = 0;
 616     int j = 0;
 617
 618     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 619         return 0;
 620     s0 = *src;
 621     while (*s0)
 622     {
 623         if (*s0 == '*')
 624         {
 625             dst[i++] = '.';
 626             dst[i++] = '*';
 627             dst_term[j++] = *s0++;
 628         }
 629         else if (*s0 == '!')
 630         {
 631             dst[i++] = '.';
 632             dst_term[j++] = *s0++;
 633         }
 634         else
 635         {
 636             const char *s1 = s0;
 637             int q_map_match = 0;
 638             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 639                                     &q_map_match);
 640             if (space_split && **map == *CHR_SPACE)
 641                 break;
 642
 643             /* add non-space char */
 644             memcpy(dst_term+j, s1, s0 - s1);
 645             j += (s0 - s1);
 646             if (!q_map_match)
 647             {
 648                 while (s1 < s0)
 649                 {
 650                     if (strchr(REGEX_CHARS, *s1))
 651                         dst[i++] = '\\';
 652                     dst[i++] = *s1++;
 653                 }
 654             }
 655             else
 656             {
 657                 char tmpbuf[80];
 658                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 659
 660                 strcpy(dst + i, map[0]);
 661                 i += strlen(map[0]);
 662             }
 663         }
 664     }
 665     if (right_truncate)
 666     {
 667         dst[i++] = '.';
 668         dst[i++] = '*';
 669     }
 670     dst[i] = '\0';
 671
 672     dst_term[j++] = '\0';
 673     *src = s0;
 674     return i;
 675 }
 676
 677
 678 /* gen_regular_rel - generate regular expression from relation
 679  *  val:     border value (inclusive)
 680  *  islt:    1 if <=; 0 if >=.
 681  */
 682 static void gen_regular_rel(char *dst, int val, int islt)
 683 {
 684     int dst_p;
 685     int w, d, i;
 686     int pos = 0;
 687     char numstr[20];
 688
 689     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 690     if (val >= 0)
 691     {
 692         if (islt)
 693             strcpy(dst, "(-[0-9]+|(");
 694         else
 695             strcpy(dst, "((");
 696     }
 697     else
 698     {
 699         if (!islt)
 700         {
 701             strcpy(dst, "([0-9]+|-(");
 702             dst_p = strlen(dst);
 703             islt = 1;
 704         }
 705         else
 706         {
 707             strcpy(dst, "(-(");
 708             islt = 0;
 709         }
 710         val = -val;
 711     }
 712     dst_p = strlen(dst);
 713     sprintf(numstr, "%d", val);
 714     for (w = strlen(numstr); --w >= 0; pos++)
 715     {
 716         d = numstr[w];
 717         if (pos > 0)
 718         {
 719             if (islt)
 720             {
 721                 if (d == '0')
 722                     continue;
 723                 d--;
 724             }
 725             else
 726             {
 727                 if (d == '9')
 728                     continue;
 729                 d++;
 730             }
 731         }
 732
 733         strcpy(dst + dst_p, numstr);
 734         dst_p = strlen(dst) - pos - 1;
 735
 736         if (islt)
 737         {
 738             if (d != '0')
 739             {
 740                 dst[dst_p++] = '[';
 741                 dst[dst_p++] = '0';
 742                 dst[dst_p++] = '-';
 743                 dst[dst_p++] = d;
 744                 dst[dst_p++] = ']';
 745             }
 746             else
 747                 dst[dst_p++] = d;
 748         }
 749         else
 750         {
 751             if (d != '9')
 752             {
 753                 dst[dst_p++] = '[';
 754                 dst[dst_p++] = d;
 755                 dst[dst_p++] = '-';
 756                 dst[dst_p++] = '9';
 757                 dst[dst_p++] = ']';
 758             }
 759             else
 760                 dst[dst_p++] = d;
 761         }
 762         for (i = 0; i<pos; i++)
 763         {
 764             dst[dst_p++] = '[';
 765             dst[dst_p++] = '0';
 766             dst[dst_p++] = '-';
 767             dst[dst_p++] = '9';
 768             dst[dst_p++] = ']';
 769         }
 770         dst[dst_p++] = '|';
 771     }
 772     dst[dst_p] = '\0';
 773     if (islt)
 774     {
 775         /* match everything less than 10^(pos-1) */
 776         strcat(dst, "0*");
 777         for (i = 1; i<pos; i++)
 778             strcat(dst, "[0-9]?");
 779     }
 780     else
 781     {
 782         /* match everything greater than 10^pos */
 783         for (i = 0; i <= pos; i++)
 784             strcat(dst, "[0-9]");
 785         strcat(dst, "[0-9]*");
 786     }
 787     strcat(dst, "))");
 788 }
 789
 790 void string_rel_add_char(char **term_p, const char *src, int *indx)
 791 {
 792     if (src[*indx] == '\\')
 793         *(*term_p)++ = src[(*indx)++];
 794     *(*term_p)++ = src[(*indx)++];
 795 }
 796
 797 /*
 798  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 799  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 800  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 801  *              ([^-a].*|a[^-b].*|ab[c-].*)
 802  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 803  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 804  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 805  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 806  */
 807 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 808                            const char **term_sub, char *term_dict,
 809                            oid_value attributeSet,
 810                            int reg_type, int space_split, char *term_dst,
 811                            int *error_code)
 812 {
 813     AttrType relation;
 814     int relation_value;
 815     int i;
 816     char *term_tmp = term_dict + strlen(term_dict);
 817     char term_component[2*IT_MAX_WORD+20];
 818
 819     attr_init(&relation, zapt, 2);
 820     relation_value = attr_find(&relation, NULL);
 821
 822     *error_code = 0;
 823     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 824     switch (relation_value)
 825     {
 826     case 1:
 827         if (!term_100(zh->reg->zebra_maps, reg_type,
 828                       term_sub, term_component,
 829                       space_split, term_dst))
 830             return 0;
 831         yaz_log(log_level_rpn, "Relation <");
 832
 833         *term_tmp++ = '(';
 834         for (i = 0; term_component[i]; )
 835         {
 836             int j = 0;
 837
 838             if (i)
 839                 *term_tmp++ = '|';
 840             while (j < i)
 841                 string_rel_add_char(&term_tmp, term_component, &j);
 842
 843             *term_tmp++ = '[';
 844
 845             *term_tmp++ = '^';
 846             string_rel_add_char(&term_tmp, term_component, &i);
 847             *term_tmp++ = '-';
 848
 849             *term_tmp++ = ']';
 850             *term_tmp++ = '.';
 851             *term_tmp++ = '*';
 852
 853             if ((term_tmp - term_dict) > IT_MAX_WORD)
 854                 break;
 855         }
 856         *term_tmp++ = ')';
 857         *term_tmp = '\0';
 858         break;
 859     case 2:
 860         if (!term_100(zh->reg->zebra_maps, reg_type,
 861                       term_sub, term_component,
 862                       space_split, term_dst))
 863             return 0;
 864         yaz_log(log_level_rpn, "Relation <=");
 865
 866         *term_tmp++ = '(';
 867         for (i = 0; term_component[i]; )
 868         {
 869             int j = 0;
 870
 871             while (j < i)
 872                 string_rel_add_char(&term_tmp, term_component, &j);
 873             *term_tmp++ = '[';
 874
 875             *term_tmp++ = '^';
 876             string_rel_add_char(&term_tmp, term_component, &i);
 877             *term_tmp++ = '-';
 878
 879             *term_tmp++ = ']';
 880             *term_tmp++ = '.';
 881             *term_tmp++ = '*';
 882
 883             *term_tmp++ = '|';
 884
 885             if ((term_tmp - term_dict) > IT_MAX_WORD)
 886                 break;
 887         }
 888         for (i = 0; term_component[i]; )
 889             string_rel_add_char(&term_tmp, term_component, &i);
 890         *term_tmp++ = ')';
 891         *term_tmp = '\0';
 892         break;
 893     case 5:
 894         if (!term_100 (zh->reg->zebra_maps, reg_type,
 895                        term_sub, term_component, space_split, term_dst))
 896             return 0;
 897         yaz_log(log_level_rpn, "Relation >");
 898
 899         *term_tmp++ = '(';
 900         for (i = 0; term_component[i];)
 901         {
 902             int j = 0;
 903
 904             while (j < i)
 905                 string_rel_add_char(&term_tmp, term_component, &j);
 906             *term_tmp++ = '[';
 907
 908             *term_tmp++ = '^';
 909             *term_tmp++ = '-';
 910             string_rel_add_char(&term_tmp, term_component, &i);
 911
 912             *term_tmp++ = ']';
 913             *term_tmp++ = '.';
 914             *term_tmp++ = '*';
 915
 916             *term_tmp++ = '|';
 917
 918             if ((term_tmp - term_dict) > IT_MAX_WORD)
 919                 break;
 920         }
 921         for (i = 0; term_component[i];)
 922             string_rel_add_char(&term_tmp, term_component, &i);
 923         *term_tmp++ = '.';
 924         *term_tmp++ = '+';
 925         *term_tmp++ = ')';
 926         *term_tmp = '\0';
 927         break;
 928     case 4:
 929         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 930                       term_component, space_split, term_dst))
 931             return 0;
 932         yaz_log(log_level_rpn, "Relation >=");
 933
 934         *term_tmp++ = '(';
 935         for (i = 0; term_component[i];)
 936         {
 937             int j = 0;
 938
 939             if (i)
 940                 *term_tmp++ = '|';
 941             while (j < i)
 942                 string_rel_add_char(&term_tmp, term_component, &j);
 943             *term_tmp++ = '[';
 944
 945             if (term_component[i+1])
 946             {
 947                 *term_tmp++ = '^';
 948                 *term_tmp++ = '-';
 949                 string_rel_add_char(&term_tmp, term_component, &i);
 950             }
 951             else
 952             {
 953                 string_rel_add_char(&term_tmp, term_component, &i);
 954                 *term_tmp++ = '-';
 955             }
 956             *term_tmp++ = ']';
 957             *term_tmp++ = '.';
 958             *term_tmp++ = '*';
 959
 960             if ((term_tmp - term_dict) > IT_MAX_WORD)
 961                 break;
 962         }
 963         *term_tmp++ = ')';
 964         *term_tmp = '\0';
 965         break;
 966     case 3:
 967     case 102:
 968     case -1:
 969         yaz_log(log_level_rpn, "Relation =");
 970         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 971                       term_component, space_split, term_dst))
 972             return 0;
 973         strcat(term_tmp, "(");
 974         strcat(term_tmp, term_component);
 975         strcat(term_tmp, ")");
 976         break;
 977     default:
 978         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 979         return 0;
 980     }
 981     return 1;
 982 }
 983
 984 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 985                              const char **term_sub,
 986                              oid_value attributeSet, NMEM stream,
 987                              struct grep_info *grep_info,
 988                              int reg_type, int complete_flag,
 989                              int num_bases, char **basenames,
 990                              char *term_dst, int xpath_use,
 991                              struct ord_list **ol);
 992
 993 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 994                                  Z_AttributesPlusTerm *zapt,
 995                                  zint *hits_limit_value,
 996                                  const char **term_ref_id_str,
 997                                  NMEM nmem)
 998 {
 999     AttrType term_ref_id_attr;
1000     AttrType hits_limit_attr;
1001     int term_ref_id_int;
1002
1003     attr_init(&hits_limit_attr, zapt, 9);
1004     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
1005
1006     attr_init(&term_ref_id_attr, zapt, 10);
1007     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
1008     if (term_ref_id_int >= 0)
1009     {
1010         char *res = nmem_malloc(nmem, 20);
1011         sprintf(res, "%d", term_ref_id_int);
1012         *term_ref_id_str = res;
1013     }
1014
1015     /* no limit given ? */
1016     if (*hits_limit_value == -1)
1017     {
1018         if (*term_ref_id_str)
1019         {
1020             /* use global if term_ref is present */
1021             *hits_limit_value = zh->approx_limit;
1022         }
1023         else
1024         {
1025             /* no counting if term_ref is not present */
1026             *hits_limit_value = 0;
1027         }
1028     }
1029     else if (*hits_limit_value == 0)
1030     {
1031         /* 0 is the same as global limit */
1032         *hits_limit_value = zh->approx_limit;
1033     }
1034     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
1035             *term_ref_id_str ? *term_ref_id_str : "none",
1036             *hits_limit_value);
1037     return ZEBRA_OK;
1038 }
1039
1040 static ZEBRA_RES term_trunc(ZebraHandle zh,
1041                             Z_AttributesPlusTerm *zapt,
1042                             const char **term_sub,
1043                             oid_value attributeSet, NMEM stream,
1044                             struct grep_info *grep_info,
1045                             int reg_type, int complete_flag,
1046                             int num_bases, char **basenames,
1047                             char *term_dst,
1048                             const char *rank_type, int xpath_use,
1049                             NMEM rset_nmem,
1050                             RSET *rset,
1051                             struct rset_key_control *kc)
1052 {
1053     ZEBRA_RES res;
1054     struct ord_list *ol;
1055     zint hits_limit_value;
1056     const char *term_ref_id_str = 0;
1057     *rset = 0;
1058
1059     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str,
1060                     stream);
1061     grep_info->isam_p_indx = 0;
1062     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
1063                       reg_type, complete_flag, num_bases, basenames,
1064                       term_dst, xpath_use, &ol);
1065     if (res != ZEBRA_OK)
1066         return res;
1067     if (!*term_sub)  /* no more terms ? */
1068         return res;
1069     yaz_log(log_level_rpn, "term: %s", term_dst);
1070     *rset = rset_trunc(zh, grep_info->isam_p_buf,
1071                        grep_info->isam_p_indx, term_dst,
1072                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1073                        zapt->term->which, rset_nmem,
1074                        kc, kc->scope, ol, reg_type, hits_limit_value,
1075                        term_ref_id_str);
1076     if (!*rset)
1077         return ZEBRA_FAIL;
1078     return ZEBRA_OK;
1079 }
1080
1081 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1082                              const char **term_sub,
1083                              oid_value attributeSet, NMEM stream,
1084                              struct grep_info *grep_info,
1085                              int reg_type, int complete_flag,
1086                              int num_bases, char **basenames,
1087                              char *term_dst, int xpath_use,
1088                              struct ord_list **ol)
1089 {
1090     char term_dict[2*IT_MAX_WORD+4000];
1091     int j, r, base_no;
1092     AttrType truncation;
1093     int truncation_value;
1094     AttrType use;
1095     int use_value;
1096     const char *use_string = 0;
1097     oid_value curAttributeSet = attributeSet;
1098     const char *termp;
1099     struct rpn_char_map_info rcmi;
1100     int space_split = complete_flag ? 0 : 1;
1101
1102     int bases_ok = 0;     /* no of databases with OK attribute */
1103
1104     *ol = ord_list_create(stream);
1105
1106     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1107     attr_init(&use, zapt, 1);
1108     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1109     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1110     attr_init(&truncation, zapt, 5);
1111     truncation_value = attr_find(&truncation, NULL);
1112     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1113
1114     if (use_value == -1)    /* no attribute - assumy "any" */
1115         use_value = 1016;
1116     for (base_no = 0; base_no < num_bases; base_no++)
1117     {
1118         int ord = -1;
1119         int attr_ok = 0;
1120         int regex_range = 0;
1121         int init_pos = 0;
1122         attent attp;
1123         data1_local_attribute id_xpath_attr;
1124         data1_local_attribute *local_attr;
1125         int max_pos, prefix_len = 0;
1126         int relation_error;
1127
1128         termp = *term_sub;
1129
1130         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1131         {
1132             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1133                            basenames[base_no]);
1134             return ZEBRA_FAIL;
1135         }
1136         if (xpath_use > 0 && use_value == -2)
1137         {
1138             /* xpath mode and we have a string attribute */
1139             attp.local_attributes = &id_xpath_attr;
1140             attp.attset_ordinal = VAL_IDXPATH;
1141             id_xpath_attr.next = 0;
1142
1143             use_value = xpath_use;  /* xpath_use as use-attribute now */
1144             id_xpath_attr.local = use_value;
1145         }
1146         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1147         {
1148             /* X-Path attribute, use numeric value directly */
1149             attp.local_attributes = &id_xpath_attr;
1150             attp.attset_ordinal = VAL_IDXPATH;
1151             id_xpath_attr.next = 0;
1152             id_xpath_attr.local = use_value;
1153         }
1154         else if (use_string &&
1155                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1156                                                      reg_type,
1157                                                      use_string)) >= 0)
1158         {
1159             /* we have a match for a raw string attribute */
1160             char ord_buf[32];
1161             int i, ord_len;
1162
1163             if (prefix_len)
1164                 term_dict[prefix_len++] = '|';
1165             else
1166                 term_dict[prefix_len++] = '(';
1167
1168             ord_len = key_SU_encode (ord, ord_buf);
1169             for (i = 0; i<ord_len; i++)
1170             {
1171                 term_dict[prefix_len++] = 1;
1172                 term_dict[prefix_len++] = ord_buf[i];
1173             }
1174             attp.local_attributes = 0;  /* no more attributes */
1175             *ol = ord_list_append(stream, *ol, ord);
1176         }
1177         else
1178         {
1179             /* lookup in the .att files . Allow string as well */
1180             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1181                                       use_string)))
1182             {
1183                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1184                         curAttributeSet, use_value, r);
1185                 if (r == -1)
1186                 {
1187                     /* set was found, but value wasn't defined */
1188                     if (use_string)
1189                         zebra_setError(zh,
1190                                        YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1191                                        use_string);
1192                     else
1193                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1194                                             use_value);
1195                 }
1196                 else
1197                 {
1198                     int oid[OID_SIZE];
1199                     struct oident oident;
1200
1201                     oident.proto = PROTO_Z3950;
1202                     oident.oclass = CLASS_ATTSET;
1203                     oident.value = curAttributeSet;
1204                     oid_ent_to_oid (&oident, oid);
1205
1206                     zebra_setError(zh,
1207                                    YAZ_BIB1_UNSUPP_ATTRIBUTE_SET,
1208                                    oident.desc);
1209
1210                 }
1211                 continue;
1212             }
1213         }
1214         for (local_attr = attp.local_attributes; local_attr;
1215              local_attr = local_attr->next)
1216         {
1217             char ord_buf[32];
1218             int i, ord_len;
1219
1220             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1221                                               reg_type,
1222                                               attp.attset_ordinal,
1223                                               local_attr->local);
1224             if (ord < 0)
1225                 continue;
1226             *ol = ord_list_append(stream, *ol, ord);
1227             if (prefix_len)
1228                 term_dict[prefix_len++] = '|';
1229             else
1230                 term_dict[prefix_len++] = '(';
1231
1232             ord_len = key_SU_encode (ord, ord_buf);
1233             for (i = 0; i<ord_len; i++)
1234             {
1235                 term_dict[prefix_len++] = 1;
1236                 term_dict[prefix_len++] = ord_buf[i];
1237             }
1238         }
1239         bases_ok++;
1240         if (prefix_len)
1241             attr_ok = 1;
1242
1243         term_dict[prefix_len++] = ')';
1244         term_dict[prefix_len] = '\0';
1245         j = prefix_len;
1246         switch (truncation_value)
1247         {
1248         case -1:         /* not specified */
1249         case 100:        /* do not truncate */
1250             if (!string_relation (zh, zapt, &termp, term_dict,
1251                                   attributeSet,
1252                                   reg_type, space_split, term_dst,
1253                                   &relation_error))
1254             {
1255                 if (relation_error)
1256                 {
1257                     zebra_setError(zh, relation_error, 0);
1258                     return ZEBRA_FAIL;
1259                 }
1260                 *term_sub = 0;
1261                 return ZEBRA_OK;
1262             }
1263             break;
1264         case 1:          /* right truncation */
1265             term_dict[j++] = '(';
1266             if (!term_100(zh->reg->zebra_maps, reg_type,
1267                           &termp, term_dict + j, space_split, term_dst))
1268             {
1269                 *term_sub = 0;
1270                 return ZEBRA_OK;
1271             }
1272             strcat(term_dict, ".*)");
1273             break;
1274         case 2:          /* keft truncation */
1275             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1276             if (!term_100(zh->reg->zebra_maps, reg_type,
1277                           &termp, term_dict + j, space_split, term_dst))
1278             {
1279                 *term_sub = 0;
1280                 return ZEBRA_OK;
1281             }
1282             strcat(term_dict, ")");
1283             break;
1284         case 3:          /* left&right truncation */
1285             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1286             if (!term_100(zh->reg->zebra_maps, reg_type,
1287                           &termp, term_dict + j, space_split, term_dst))
1288             {
1289                 *term_sub = 0;
1290                 return ZEBRA_OK;
1291             }
1292             strcat(term_dict, ".*)");
1293             break;
1294         case 101:        /* process # in term */
1295             term_dict[j++] = '(';
1296             if (!term_101(zh->reg->zebra_maps, reg_type,
1297                           &termp, term_dict + j, space_split, term_dst))
1298             {
1299                 *term_sub = 0;
1300                 return ZEBRA_OK;
1301             }
1302             strcat(term_dict, ")");
1303             break;
1304         case 102:        /* Regexp-1 */
1305             term_dict[j++] = '(';
1306             if (!term_102(zh->reg->zebra_maps, reg_type,
1307                           &termp, term_dict + j, space_split, term_dst))
1308             {
1309                 *term_sub = 0;
1310                 return ZEBRA_OK;
1311             }
1312             strcat(term_dict, ")");
1313             break;
1314         case 103:       /* Regexp-2 */
1315             regex_range = 1;
1316             term_dict[j++] = '(';
1317             init_pos = 2;
1318             if (!term_103(zh->reg->zebra_maps, reg_type,
1319                           &termp, term_dict + j, &regex_range,
1320                           space_split, term_dst))
1321             {
1322                 *term_sub = 0;
1323                 return ZEBRA_OK;
1324             }
1325             strcat(term_dict, ")");
1326             break;
1327         case 104:        /* process # and ! in term */
1328             term_dict[j++] = '(';
1329             if (!term_104(zh->reg->zebra_maps, reg_type,
1330                           &termp, term_dict + j, space_split, term_dst))
1331             {
1332                 *term_sub = 0;
1333                 return ZEBRA_OK;
1334             }
1335             strcat(term_dict, ")");
1336             break;
1337         case 105:        /* process * and ! in term */
1338             term_dict[j++] = '(';
1339             if (!term_105(zh->reg->zebra_maps, reg_type,
1340                           &termp, term_dict + j, space_split, term_dst, 1))
1341             {
1342                 *term_sub = 0;
1343                 return ZEBRA_OK;
1344             }
1345             strcat(term_dict, ")");
1346             break;
1347         case 106:        /* process * and ! in term */
1348             term_dict[j++] = '(';
1349             if (!term_105(zh->reg->zebra_maps, reg_type,
1350                           &termp, term_dict + j, space_split, term_dst, 0))
1351             {
1352                 *term_sub = 0;
1353                 return ZEBRA_OK;
1354             }
1355             strcat(term_dict, ")");
1356             break;
1357         default:
1358             zebra_setError_zint(zh,
1359                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1360                                 truncation_value);
1361             return ZEBRA_FAIL;
1362         }
1363         if (attr_ok)
1364         {
1365             char buf[80];
1366             const char *input = term_dict + prefix_len;
1367             esc_str(buf, sizeof(buf), input, strlen(input));
1368         }
1369         if (attr_ok)
1370         {
1371             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1372             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1373                                  grep_info, &max_pos, init_pos,
1374                                  grep_handle);
1375             if (r)
1376                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1377         }
1378     }
1379     if (!bases_ok)
1380         return ZEBRA_FAIL;
1381     *term_sub = termp;
1382     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1383     return ZEBRA_OK;
1384 }
1385
1386
1387 /* convert APT search term to UTF8 */
1388 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1389                                    char *termz)
1390 {
1391     size_t sizez;
1392     Z_Term *term = zapt->term;
1393
1394     switch (term->which)
1395     {
1396     case Z_Term_general:
1397         if (zh->iconv_to_utf8 != 0)
1398         {
1399             char *inbuf = (char *) term->u.general->buf;
1400             size_t inleft = term->u.general->len;
1401             char *outbuf = termz;
1402             size_t outleft = IT_MAX_WORD-1;
1403             size_t ret;
1404
1405             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1406                         &outbuf, &outleft);
1407             if (ret == (size_t)(-1))
1408             {
1409                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1410                 zebra_setError(
1411                     zh,
1412                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1413                     0);
1414                 return ZEBRA_FAIL;
1415             }
1416             *outbuf = 0;
1417         }
1418         else
1419         {
1420             sizez = term->u.general->len;
1421             if (sizez > IT_MAX_WORD-1)
1422                 sizez = IT_MAX_WORD-1;
1423             memcpy (termz, term->u.general->buf, sizez);
1424             termz[sizez] = '\0';
1425         }
1426         break;
1427     case Z_Term_characterString:
1428         sizez = strlen(term->u.characterString);
1429         if (sizez > IT_MAX_WORD-1)
1430             sizez = IT_MAX_WORD-1;
1431         memcpy (termz, term->u.characterString, sizez);
1432         termz[sizez] = '\0';
1433         break;
1434     default:
1435         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1436         return ZEBRA_FAIL;
1437     }
1438     return ZEBRA_OK;
1439 }
1440
1441 /* convert APT SCAN term to internal cmap */
1442 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1443                                  char *termz, int reg_type)
1444 {
1445     char termz0[IT_MAX_WORD];
1446
1447     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1448         return ZEBRA_FAIL;    /* error */
1449     else
1450     {
1451         const char **map;
1452         const char *cp = (const char *) termz0;
1453         const char *cp_end = cp + strlen(cp);
1454         const char *src;
1455         int i = 0;
1456         const char *space_map = NULL;
1457         int len;
1458
1459         while ((len = (cp_end - cp)) > 0)
1460         {
1461             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1462             if (**map == *CHR_SPACE)
1463                 space_map = *map;
1464             else
1465             {
1466                 if (i && space_map)
1467                     for (src = space_map; *src; src++)
1468                         termz[i++] = *src;
1469                 space_map = NULL;
1470                 for (src = *map; *src; src++)
1471                     termz[i++] = *src;
1472             }
1473         }
1474         termz[i] = '\0';
1475     }
1476     return ZEBRA_OK;
1477 }
1478
1479 static void grep_info_delete(struct grep_info *grep_info)
1480 {
1481 #ifdef TERM_COUNT
1482     xfree(grep_info->term_no);
1483 #endif
1484     xfree(grep_info->isam_p_buf);
1485 }
1486
1487 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1488                                    Z_AttributesPlusTerm *zapt,
1489                                    struct grep_info *grep_info,
1490                                    int reg_type)
1491 {
1492     AttrType termset;
1493     int termset_value_numeric;
1494     const char *termset_value_string;
1495
1496 #ifdef TERM_COUNT
1497     grep_info->term_no = 0;
1498 #endif
1499     grep_info->isam_p_size = 0;
1500     grep_info->isam_p_buf = NULL;
1501     grep_info->zh = zh;
1502     grep_info->reg_type = reg_type;
1503     grep_info->termset = 0;
1504
1505     if (!zapt)
1506         return ZEBRA_OK;
1507     attr_init(&termset, zapt, 8);
1508     termset_value_numeric =
1509         attr_find_ex(&termset, NULL, &termset_value_string);
1510     if (termset_value_numeric != -1)
1511     {
1512         char resname[32];
1513         const char *termset_name = 0;
1514         if (termset_value_numeric != -2)
1515         {
1516
1517             sprintf(resname, "%d", termset_value_numeric);
1518             termset_name = resname;
1519         }
1520         else
1521             termset_name = termset_value_string;
1522         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1523         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1524         if (!grep_info->termset)
1525         {
1526             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1527             return ZEBRA_FAIL;
1528         }
1529     }
1530     return ZEBRA_OK;
1531 }
1532
1533 /**
1534   \brief Create result set(s) for list of terms
1535   \param zh Zebra Handle
1536   \param termz term as used in query but converted to UTF-8
1537   \param attributeSet default attribute set
1538   \param stream memory for result
1539   \param reg_type register type ('w', 'p',..)
1540   \param complete_flag whether it's phrases or not
1541   \param rank_type term flags for ranking
1542   \param xpath_use use attribute for X-Path (-1 for no X-path)
1543   \param num_bases number of databases
1544   \param basenames array of databases
1545   \param rset_mem memory for result sets
1546   \param result_sets output result set for each term in list (output)
1547   \param number number of output result sets
1548   \param kc rset key control to be used for created result sets
1549 */
1550 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1551                                  Z_AttributesPlusTerm *zapt,
1552                                  const char *termz,
1553                                  oid_value attributeSet,
1554                                  NMEM stream,
1555                                  int reg_type, int complete_flag,
1556                                  const char *rank_type, int xpath_use,
1557                                  int num_bases, char **basenames,
1558                                  NMEM rset_nmem,
1559                                  RSET **result_sets, int *num_result_sets,
1560                                  struct rset_key_control *kc)
1561 {
1562     char term_dst[IT_MAX_WORD+1];
1563     struct grep_info grep_info;
1564     const char *termp = termz;
1565     int alloc_sets = 0;
1566
1567     *num_result_sets = 0;
1568     *term_dst = 0;
1569     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1570         return ZEBRA_FAIL;
1571     while(1)
1572     {
1573         ZEBRA_RES res;
1574
1575         if (alloc_sets == *num_result_sets)
1576         {
1577             int add = 10;
1578             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1579                                               sizeof(*rnew));
1580             if (alloc_sets)
1581                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1582             alloc_sets = alloc_sets + add;
1583             *result_sets = rnew;
1584         }
1585         res = term_trunc(zh, zapt, &termp, attributeSet,
1586                          stream, &grep_info,
1587                          reg_type, complete_flag,
1588                          num_bases, basenames,
1589                          term_dst, rank_type,
1590                          xpath_use, rset_nmem,
1591                          &(*result_sets)[*num_result_sets],
1592                          kc);
1593         if (res != ZEBRA_OK)
1594         {
1595             int i;
1596             for (i = 0; i < *num_result_sets; i++)
1597                 rset_delete((*result_sets)[i]);
1598             grep_info_delete (&grep_info);
1599             return res;
1600         }
1601         if ((*result_sets)[*num_result_sets] == 0)
1602             break;
1603         (*num_result_sets)++;
1604     }
1605     grep_info_delete(&grep_info);
1606     return ZEBRA_OK;
1607 }
1608
1609 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1610                                        Z_AttributesPlusTerm *zapt,
1611                                        const char *termz_org,
1612                                        oid_value attributeSet,
1613                                        NMEM stream,
1614                                        int reg_type, int complete_flag,
1615                                        const char *rank_type, int xpath_use,
1616                                        int num_bases, char **basenames,
1617                                        NMEM rset_nmem,
1618                                        RSET *rset,
1619                                        struct rset_key_control *kc)
1620 {
1621     RSET *result_sets = 0;
1622     int num_result_sets = 0;
1623     ZEBRA_RES res =
1624         term_list_trunc(zh, zapt, termz_org, attributeSet,
1625                         stream, reg_type, complete_flag,
1626                         rank_type, xpath_use,
1627                         num_bases, basenames,
1628                         rset_nmem,
1629                         &result_sets, &num_result_sets, kc);
1630     if (res != ZEBRA_OK)
1631         return res;
1632     if (num_result_sets == 0)
1633         *rset = rsnull_create (rset_nmem, kc, 0);
1634     else if (num_result_sets == 1)
1635         *rset = result_sets[0];
1636     else
1637         *rset = rsprox_create(rset_nmem, kc, kc->scope,
1638                               num_result_sets, result_sets,
1639                               1 /* ordered */, 0 /* exclusion */,
1640                               3 /* relation */, 1 /* distance */);
1641     if (!*rset)
1642         return ZEBRA_FAIL;
1643     return ZEBRA_OK;
1644 }
1645
1646 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1647                                         Z_AttributesPlusTerm *zapt,
1648                                         const char *termz_org,
1649                                         oid_value attributeSet,
1650                                         NMEM stream,
1651                                         int reg_type, int complete_flag,
1652                                         const char *rank_type,
1653                                         int xpath_use,
1654                                         int num_bases, char **basenames,
1655                                         NMEM rset_nmem,
1656                                         RSET *rset,
1657                                         struct rset_key_control *kc)
1658 {
1659     RSET *result_sets = 0;
1660     int num_result_sets = 0;
1661     ZEBRA_RES res =
1662         term_list_trunc(zh, zapt, termz_org, attributeSet,
1663                         stream, reg_type, complete_flag,
1664                         rank_type, xpath_use,
1665                         num_bases, basenames,
1666                         rset_nmem,
1667                         &result_sets, &num_result_sets, kc);
1668     if (res != ZEBRA_OK)
1669         return res;
1670     if (num_result_sets == 0)
1671         *rset = rsnull_create (rset_nmem, kc, 0);
1672     else if (num_result_sets == 1)
1673         *rset = result_sets[0];
1674     else
1675         *rset = rsmulti_or_create(rset_nmem, kc, kc->scope, 0 /* termid */,
1676                                   num_result_sets, result_sets);
1677     if (!*rset)
1678         return ZEBRA_FAIL;
1679     return ZEBRA_OK;
1680 }
1681
1682 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1683                                          Z_AttributesPlusTerm *zapt,
1684                                          const char *termz_org,
1685                                          oid_value attributeSet,
1686                                          NMEM stream,
1687                                          int reg_type, int complete_flag,
1688                                          const char *rank_type,
1689                                          int xpath_use,
1690                                          int num_bases, char **basenames,
1691                                          NMEM rset_nmem,
1692                                          RSET *rset,
1693                                          struct rset_key_control *kc)
1694 {
1695     RSET *result_sets = 0;
1696     int num_result_sets = 0;
1697     ZEBRA_RES res =
1698         term_list_trunc(zh, zapt, termz_org, attributeSet,
1699                         stream, reg_type, complete_flag,
1700                         rank_type, xpath_use,
1701                         num_bases, basenames,
1702                         rset_nmem,
1703                         &result_sets, &num_result_sets,
1704                         kc);
1705     if (res != ZEBRA_OK)
1706         return res;
1707     if (num_result_sets == 0)
1708         *rset = rsnull_create (rset_nmem, kc, 0);
1709     else if (num_result_sets == 1)
1710         *rset = result_sets[0];
1711     else
1712         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1713                                    num_result_sets, result_sets);
1714     if (!*rset)
1715         return ZEBRA_FAIL;
1716     return ZEBRA_OK;
1717 }
1718
1719 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1720                             const char **term_sub,
1721                             char *term_dict,
1722                             oid_value attributeSet,
1723                             struct grep_info *grep_info,
1724                             int *max_pos,
1725                             int reg_type,
1726                             char *term_dst,
1727                             int *error_code)
1728 {
1729     AttrType relation;
1730     int relation_value;
1731     int term_value;
1732     int r;
1733     char *term_tmp = term_dict + strlen(term_dict);
1734
1735     *error_code = 0;
1736     attr_init(&relation, zapt, 2);
1737     relation_value = attr_find(&relation, NULL);
1738
1739     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1740
1741     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1742                   term_dst))
1743         return 0;
1744     term_value = atoi (term_tmp);
1745     switch (relation_value)
1746     {
1747     case 1:
1748         yaz_log(log_level_rpn, "Relation <");
1749         gen_regular_rel(term_tmp, term_value-1, 1);
1750         break;
1751     case 2:
1752         yaz_log(log_level_rpn, "Relation <=");
1753         gen_regular_rel(term_tmp, term_value, 1);
1754         break;
1755     case 4:
1756         yaz_log(log_level_rpn, "Relation >=");
1757         gen_regular_rel(term_tmp, term_value, 0);
1758         break;
1759     case 5:
1760         yaz_log(log_level_rpn, "Relation >");
1761         gen_regular_rel(term_tmp, term_value+1, 0);
1762         break;
1763     case -1:
1764     case 3:
1765         yaz_log(log_level_rpn, "Relation =");
1766         sprintf(term_tmp, "(0*%d)", term_value);
1767         break;
1768     default:
1769         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1770         return 0;
1771     }
1772     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1773     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1774                           0, grep_handle);
1775     if (r)
1776         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1777     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1778     return 1;
1779 }
1780
1781 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1782                               const char **term_sub,
1783                               oid_value attributeSet,
1784                               struct grep_info *grep_info,
1785                               int reg_type, int complete_flag,
1786                               int num_bases, char **basenames,
1787                               char *term_dst, int xpath_use, NMEM stream)
1788 {
1789     char term_dict[2*IT_MAX_WORD+2];
1790     int r, base_no;
1791     AttrType use;
1792     int use_value;
1793     const char *use_string = 0;
1794     oid_value curAttributeSet = attributeSet;
1795     const char *termp;
1796     struct rpn_char_map_info rcmi;
1797
1798     int bases_ok = 0;     /* no of databases with OK attribute */
1799
1800     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1801     attr_init(&use, zapt, 1);
1802     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1803
1804     if (use_value == -1)
1805         use_value = 1016;
1806
1807     for (base_no = 0; base_no < num_bases; base_no++)
1808     {
1809         attent attp;
1810         data1_local_attribute id_xpath_attr;
1811         data1_local_attribute *local_attr;
1812         int max_pos, prefix_len = 0;
1813         int relation_error = 0;
1814
1815         termp = *term_sub;
1816         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1817         {
1818             use_value = xpath_use;
1819             attp.local_attributes = &id_xpath_attr;
1820             attp.attset_ordinal = VAL_IDXPATH;
1821             id_xpath_attr.next = 0;
1822             id_xpath_attr.local = use_value;
1823         }
1824         else if (curAttributeSet == VAL_IDXPATH)
1825         {
1826             attp.local_attributes = &id_xpath_attr;
1827             attp.attset_ordinal = VAL_IDXPATH;
1828             id_xpath_attr.next = 0;
1829             id_xpath_attr.local = use_value;
1830         }
1831         else
1832         {
1833             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1834                                       use_string)))
1835             {
1836                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1837                       curAttributeSet, use_value, r);
1838                 if (r == -1)
1839                 {
1840                     if (use_string)
1841                         zebra_setError(zh,
1842                                        YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1843                                        use_string);
1844                     else
1845                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1846                                             use_value);
1847                 }
1848                 else
1849                     zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_SET, 0);
1850                 continue;
1851             }
1852         }
1853         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1854         {
1855             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1856                            basenames[base_no]);
1857             return ZEBRA_FAIL;
1858         }
1859         for (local_attr = attp.local_attributes; local_attr;
1860              local_attr = local_attr->next)
1861         {
1862             int ord;
1863             char ord_buf[32];
1864             int i, ord_len;
1865
1866             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1867                                               reg_type,
1868                                               attp.attset_ordinal,
1869                                               local_attr->local);
1870             if (ord < 0)
1871                 continue;
1872             if (prefix_len)
1873                 term_dict[prefix_len++] = '|';
1874             else
1875                 term_dict[prefix_len++] = '(';
1876
1877             ord_len = key_SU_encode (ord, ord_buf);
1878             for (i = 0; i<ord_len; i++)
1879             {
1880                 term_dict[prefix_len++] = 1;
1881                 term_dict[prefix_len++] = ord_buf[i];
1882             }
1883         }
1884         if (!prefix_len)
1885         {
1886             zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE, use_value);
1887             continue;
1888         }
1889         bases_ok++;
1890         term_dict[prefix_len++] = ')';
1891         term_dict[prefix_len] = '\0';
1892         if (!numeric_relation(zh, zapt, &termp, term_dict,
1893                               attributeSet, grep_info, &max_pos, reg_type,
1894                               term_dst, &relation_error))
1895         {
1896             if (relation_error)
1897             {
1898                 zebra_setError(zh, relation_error, 0);
1899                 return ZEBRA_FAIL;
1900             }
1901             *term_sub = 0;
1902             return ZEBRA_OK;
1903         }
1904     }
1905     if (!bases_ok)
1906         return ZEBRA_FAIL;
1907     *term_sub = termp;
1908     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1909     return ZEBRA_OK;
1910 }
1911
1912
1913 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1914                                         Z_AttributesPlusTerm *zapt,
1915                                         const char *termz,
1916                                         oid_value attributeSet,
1917                                         NMEM stream,
1918                                         int reg_type, int complete_flag,
1919                                         const char *rank_type, int xpath_use,
1920                                         int num_bases, char **basenames,
1921                                         NMEM rset_nmem,
1922                                         RSET *rset,
1923                                         struct rset_key_control *kc)
1924 {
1925     char term_dst[IT_MAX_WORD+1];
1926     const char *termp = termz;
1927     RSET *result_sets = 0;
1928     int num_result_sets = 0;
1929     ZEBRA_RES res;
1930     struct grep_info grep_info;
1931     int alloc_sets = 0;
1932     zint hits_limit_value;
1933     const char *term_ref_id_str = 0;
1934
1935     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1936
1937     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1938     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1939         return ZEBRA_FAIL;
1940     while (1)
1941     {
1942         if (alloc_sets == num_result_sets)
1943         {
1944             int add = 10;
1945             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1946                                               sizeof(*rnew));
1947             if (alloc_sets)
1948                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1949             alloc_sets = alloc_sets + add;
1950             result_sets = rnew;
1951         }
1952         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1953         grep_info.isam_p_indx = 0;
1954         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1955                            reg_type, complete_flag, num_bases, basenames,
1956                            term_dst, xpath_use,
1957                            stream);
1958         if (res == ZEBRA_FAIL || termp == 0)
1959             break;
1960         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1961         result_sets[num_result_sets] =
1962             rset_trunc(zh, grep_info.isam_p_buf,
1963                        grep_info.isam_p_indx, term_dst,
1964                        strlen(term_dst), rank_type,
1965                        0 /* preserve position */,
1966                        zapt->term->which, rset_nmem,
1967                        kc, kc->scope, 0, reg_type,
1968                        hits_limit_value,
1969                        term_ref_id_str);
1970         if (!result_sets[num_result_sets])
1971             break;
1972         num_result_sets++;
1973     }
1974     grep_info_delete(&grep_info);
1975     if (termp)
1976     {
1977         int i;
1978         for (i = 0; i<num_result_sets; i++)
1979             rset_delete(result_sets[i]);
1980         return ZEBRA_FAIL;
1981     }
1982     if (num_result_sets == 0)
1983         *rset = rsnull_create(rset_nmem, kc, 0);
1984     if (num_result_sets == 1)
1985         *rset = result_sets[0];
1986     else
1987         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1988                                    num_result_sets, result_sets);
1989     if (!*rset)
1990         return ZEBRA_FAIL;
1991     return ZEBRA_OK;
1992 }
1993
1994 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1995                                       Z_AttributesPlusTerm *zapt,
1996                                       const char *termz,
1997                                       oid_value attributeSet,
1998                                       NMEM stream,
1999                                       const char *rank_type, NMEM rset_nmem,
2000                                       RSET *rset,
2001                                       struct rset_key_control *kc)
2002 {
2003     RSFD rsfd;
2004     struct it_key key;
2005     int sys;
2006     *rset = rstemp_create(rset_nmem, kc, kc->scope,
2007                           res_get (zh->res, "setTmpDir"),0 );
2008     rsfd = rset_open(*rset, RSETF_WRITE);
2009
2010     sys = atoi(termz);
2011     if (sys <= 0)
2012         sys = 1;
2013     key.mem[0] = sys;
2014     key.mem[1] = 1;
2015     key.len = 2;
2016     rset_write (rsfd, &key);
2017     rset_close (rsfd);
2018     return ZEBRA_OK;
2019 }
2020
2021 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2022                                oid_value attributeSet, NMEM stream,
2023                                Z_SortKeySpecList *sort_sequence,
2024                                const char *rank_type,
2025                                NMEM rset_nmem,
2026                                RSET *rset,
2027                                struct rset_key_control *kc)
2028 {
2029     int i;
2030     int sort_relation_value;
2031     AttrType sort_relation_type;
2032     Z_SortKeySpec *sks;
2033     Z_SortKey *sk;
2034     int oid[OID_SIZE];
2035     oident oe;
2036     char termz[20];
2037
2038     attr_init(&sort_relation_type, zapt, 7);
2039     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
2040
2041     if (!sort_sequence->specs)
2042     {
2043         sort_sequence->num_specs = 10;
2044         sort_sequence->specs = (Z_SortKeySpec **)
2045             nmem_malloc(stream, sort_sequence->num_specs *
2046                          sizeof(*sort_sequence->specs));
2047         for (i = 0; i<sort_sequence->num_specs; i++)
2048             sort_sequence->specs[i] = 0;
2049     }
2050     if (zapt->term->which != Z_Term_general)
2051         i = 0;
2052     else
2053         i = atoi_n ((char *) zapt->term->u.general->buf,
2054                     zapt->term->u.general->len);
2055     if (i >= sort_sequence->num_specs)
2056         i = 0;
2057     sprintf(termz, "%d", i);
2058
2059     oe.proto = PROTO_Z3950;
2060     oe.oclass = CLASS_ATTSET;
2061     oe.value = attributeSet;
2062     if (!oid_ent_to_oid (&oe, oid))
2063         return ZEBRA_FAIL;
2064
2065     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2066     sks->sortElement = (Z_SortElement *)
2067         nmem_malloc(stream, sizeof(*sks->sortElement));
2068     sks->sortElement->which = Z_SortElement_generic;
2069     sk = sks->sortElement->u.generic = (Z_SortKey *)
2070         nmem_malloc(stream, sizeof(*sk));
2071     sk->which = Z_SortKey_sortAttributes;
2072     sk->u.sortAttributes = (Z_SortAttributes *)
2073         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2074
2075     sk->u.sortAttributes->id = oid;
2076     sk->u.sortAttributes->list = zapt->attributes;
2077
2078     sks->sortRelation = (int *)
2079         nmem_malloc(stream, sizeof(*sks->sortRelation));
2080     if (sort_relation_value == 1)
2081         *sks->sortRelation = Z_SortKeySpec_ascending;
2082     else if (sort_relation_value == 2)
2083         *sks->sortRelation = Z_SortKeySpec_descending;
2084     else
2085         *sks->sortRelation = Z_SortKeySpec_ascending;
2086
2087     sks->caseSensitivity = (int *)
2088         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2089     *sks->caseSensitivity = 0;
2090
2091     sks->which = Z_SortKeySpec_null;
2092     sks->u.null = odr_nullval ();
2093     sort_sequence->specs[i] = sks;
2094     *rset = rsnull_create (rset_nmem, kc, 0);
2095     return ZEBRA_OK;
2096 }
2097
2098
2099 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2100                        oid_value attributeSet,
2101                        struct xpath_location_step *xpath, int max, NMEM mem)
2102 {
2103     oid_value curAttributeSet = attributeSet;
2104     AttrType use;
2105     const char *use_string = 0;
2106
2107     attr_init(&use, zapt, 1);
2108     attr_find_ex(&use, &curAttributeSet, &use_string);
2109
2110     if (!use_string || *use_string != '/')
2111         return -1;
2112
2113     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2114 }
2115
2116
2117
2118 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2119                         int reg_type, const char *term, int use,
2120                         oid_value curAttributeSet, NMEM rset_nmem,
2121                         struct rset_key_control *kc)
2122 {
2123     RSET rset;
2124     struct grep_info grep_info;
2125     char term_dict[2048];
2126     char ord_buf[32];
2127     int prefix_len = 0;
2128     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, reg_type,
2129                                           curAttributeSet, use);
2130     int ord_len, i, r, max_pos;
2131     int term_type = Z_Term_characterString;
2132     const char *flags = "void";
2133
2134     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
2135         return rsnull_create(rset_nmem, kc, 0);
2136
2137     if (ord < 0)
2138         return rsnull_create(rset_nmem, kc, 0);
2139     if (prefix_len)
2140         term_dict[prefix_len++] = '|';
2141     else
2142         term_dict[prefix_len++] = '(';
2143
2144     ord_len = key_SU_encode (ord, ord_buf);
2145     for (i = 0; i<ord_len; i++)
2146     {
2147         term_dict[prefix_len++] = 1;
2148         term_dict[prefix_len++] = ord_buf[i];
2149     }
2150     term_dict[prefix_len++] = ')';
2151     strcpy(term_dict+prefix_len, term);
2152
2153     grep_info.isam_p_indx = 0;
2154     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2155                           &grep_info, &max_pos, 0, grep_handle);
2156     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2157              grep_info.isam_p_indx);
2158     rset = rset_trunc(zh, grep_info.isam_p_buf,
2159                       grep_info.isam_p_indx, term, strlen(term),
2160                       flags, 1, term_type,rset_nmem,
2161                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
2162                       0 /* term_ref_id_str */);
2163     grep_info_delete(&grep_info);
2164     return rset;
2165 }
2166
2167 static
2168 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
2169                            oid_value attributeSet,
2170                            int num_bases, char **basenames,
2171                            NMEM stream, const char *rank_type, RSET rset,
2172                            int xpath_len, struct xpath_location_step *xpath,
2173                            NMEM rset_nmem,
2174                            RSET *rset_out,
2175                            struct rset_key_control *kc)
2176 {
2177     oid_value curAttributeSet = attributeSet;
2178     int base_no;
2179     int i;
2180
2181     if (xpath_len < 0)
2182     {
2183         *rset_out = rset;
2184         return ZEBRA_OK;
2185     }
2186
2187     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2188     for (i = 0; i<xpath_len; i++)
2189     {
2190         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2191
2192     }
2193
2194     curAttributeSet = VAL_IDXPATH;
2195
2196     /*
2197       //a    ->    a/.*
2198       //a/b  ->    b/a/.*
2199       /a     ->    a/
2200       /a/b   ->    b/a/
2201
2202       /      ->    none
2203
2204    a[@attr = value]/b[@other = othervalue]
2205
2206  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2207  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2208  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2209  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2210  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2211  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2212
2213     */
2214
2215     dict_grep_cmap (zh->reg->dict, 0, 0);
2216
2217     for (base_no = 0; base_no < num_bases; base_no++)
2218     {
2219         int level = xpath_len;
2220         int first_path = 1;
2221
2222         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2223         {
2224             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2225                            basenames[base_no]);
2226             *rset_out = rset;
2227             return ZEBRA_FAIL;
2228         }
2229         while (--level >= 0)
2230         {
2231             char xpath_rev[128];
2232             int i, len;
2233             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2234
2235             *xpath_rev = 0;
2236             len = 0;
2237             for (i = level; i >= 1; --i)
2238             {
2239                 const char *cp = xpath[i].part;
2240                 if (*cp)
2241                 {
2242                     for (;*cp; cp++)
2243                         if (*cp == '*')
2244                         {
2245                             memcpy (xpath_rev + len, "[^/]*", 5);
2246                             len += 5;
2247                         }
2248                         else if (*cp == ' ')
2249                         {
2250
2251                             xpath_rev[len++] = 1;
2252                             xpath_rev[len++] = ' ';
2253                         }
2254
2255                         else
2256                             xpath_rev[len++] = *cp;
2257                     xpath_rev[len++] = '/';
2258                 }
2259                 else if (i == 1)  /* // case */
2260                 {
2261                     xpath_rev[len++] = '.';
2262                     xpath_rev[len++] = '*';
2263                 }
2264             }
2265             xpath_rev[len] = 0;
2266
2267             if (xpath[level].predicate &&
2268                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2269                 xpath[level].predicate->u.relation.name[0])
2270             {
2271                 WRBUF wbuf = wrbuf_alloc();
2272                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2273                 if (xpath[level].predicate->u.relation.value)
2274                 {
2275                     const char *cp = xpath[level].predicate->u.relation.value;
2276                     wrbuf_putc(wbuf, '=');
2277
2278                     while (*cp)
2279                     {
2280                         if (strchr(REGEX_CHARS, *cp))
2281                             wrbuf_putc(wbuf, '\\');
2282                         wrbuf_putc(wbuf, *cp);
2283                         cp++;
2284                     }
2285                 }
2286                 wrbuf_puts(wbuf, "");
2287                 rset_attr = xpath_trunc(
2288                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2289                     curAttributeSet, rset_nmem, kc);
2290                 wrbuf_free(wbuf, 1);
2291             }
2292             else
2293             {
2294                 if (!first_path)
2295                     continue;
2296             }
2297             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2298             if (strlen(xpath_rev))
2299             {
2300                 rset_start_tag = xpath_trunc(zh, stream, '0',
2301                         xpath_rev, 1, curAttributeSet, rset_nmem, kc);
2302
2303                 rset_end_tag = xpath_trunc(zh, stream, '0',
2304                         xpath_rev, 2, curAttributeSet, rset_nmem, kc);
2305
2306                 rset = rsbetween_create(rset_nmem, kc, kc->scope,
2307                                         rset_start_tag, rset,
2308                                         rset_end_tag, rset_attr);
2309             }
2310             first_path = 0;
2311         }
2312     }
2313     *rset_out = rset;
2314     return ZEBRA_OK;
2315 }
2316
2317 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2318                                 oid_value attributeSet, NMEM stream,
2319                                 Z_SortKeySpecList *sort_sequence,
2320                                 int num_bases, char **basenames,
2321                                 NMEM rset_nmem,
2322                                 RSET *rset,
2323                                 struct rset_key_control *kc)
2324 {
2325     ZEBRA_RES res = ZEBRA_OK;
2326     unsigned reg_id;
2327     char *search_type = NULL;
2328     char rank_type[128];
2329     int complete_flag;
2330     int sort_flag;
2331     char termz[IT_MAX_WORD+1];
2332     int xpath_len;
2333     int xpath_use = 0;
2334     struct xpath_location_step xpath[10];
2335
2336     if (!log_level_set)
2337     {
2338         log_level_rpn = yaz_log_module_level("rpn");
2339         log_level_set = 1;
2340     }
2341     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2342                     rank_type, &complete_flag, &sort_flag);
2343
2344     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2345     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2346     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2347     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2348
2349     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2350         return ZEBRA_FAIL;
2351
2352     if (sort_flag)
2353         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2354                              rank_type, rset_nmem, rset, kc);
2355     /* consider if an X-Path query is used */
2356     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2357     if (xpath_len >= 0)
2358     {
2359         xpath_use = 1016;  /* searching for element by default */
2360         if (xpath[xpath_len-1].part[0] == '@')
2361             xpath_use = 1015;  /* last step an attribute .. */
2362     }
2363
2364     /* search using one of the various search type strategies
2365        termz is our UTF-8 search term
2366        attributeSet is top-level default attribute set
2367        stream is ODR for search
2368        reg_id is the register type
2369        complete_flag is 1 for complete subfield, 0 for incomplete
2370        xpath_use is use-attribute to be used for X-Path search, 0 for none
2371     */
2372     if (!strcmp(search_type, "phrase"))
2373     {
2374         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2375                                     reg_id, complete_flag, rank_type,
2376                                     xpath_use,
2377                                     num_bases, basenames, rset_nmem,
2378                                     rset, kc);
2379     }
2380     else if (!strcmp(search_type, "and-list"))
2381     {
2382         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2383                                       reg_id, complete_flag, rank_type,
2384                                       xpath_use,
2385                                       num_bases, basenames, rset_nmem,
2386                                       rset, kc);
2387     }
2388     else if (!strcmp(search_type, "or-list"))
2389     {
2390         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2391                                      reg_id, complete_flag, rank_type,
2392                                      xpath_use,
2393                                      num_bases, basenames, rset_nmem,
2394                                      rset, kc);
2395     }
2396     else if (!strcmp(search_type, "local"))
2397     {
2398         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2399                                    rank_type, rset_nmem, rset, kc);
2400     }
2401     else if (!strcmp(search_type, "numeric"))
2402     {
2403         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2404                                      reg_id, complete_flag, rank_type,
2405                                      xpath_use,
2406                                      num_bases, basenames, rset_nmem,
2407                                      rset, kc);
2408     }
2409     else
2410     {
2411         zebra_setError(zh, YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE, 0);
2412         res = ZEBRA_FAIL;
2413     }
2414     if (res != ZEBRA_OK)
2415         return res;
2416     if (!*rset)
2417         return ZEBRA_FAIL;
2418     return rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2419                             stream, rank_type, *rset,
2420                             xpath_len, xpath, rset_nmem, rset, kc);
2421 }
2422
2423 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2424                                       oid_value attributeSet,
2425                                       NMEM stream, NMEM rset_nmem,
2426                                       Z_SortKeySpecList *sort_sequence,
2427                                       int num_bases, char **basenames,
2428                                       RSET **result_sets, int *num_result_sets,
2429                                       Z_Operator *parent_op,
2430                                       struct rset_key_control *kc);
2431
2432 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2433                          oid_value attributeSet,
2434                          NMEM stream, NMEM rset_nmem,
2435                          Z_SortKeySpecList *sort_sequence,
2436                          int num_bases, char **basenames,
2437                          RSET *result_set)
2438 {
2439     RSET *result_sets = 0;
2440     int num_result_sets = 0;
2441     ZEBRA_RES res;
2442     struct rset_key_control *kc = zebra_key_control_create(zh);
2443
2444     res = rpn_search_structure(zh, zs, attributeSet,
2445                                stream, rset_nmem,
2446                                sort_sequence,
2447                                num_bases, basenames,
2448                                &result_sets, &num_result_sets,
2449                                0 /* no parent op */,
2450                                kc);
2451     if (res != ZEBRA_OK)
2452     {
2453         int i;
2454         for (i = 0; i<num_result_sets; i++)
2455             rset_delete(result_sets[i]);
2456         *result_set = 0;
2457     }
2458     else
2459     {
2460         assert(num_result_sets == 1);
2461         assert(result_sets);
2462         assert(*result_sets);
2463         *result_set = *result_sets;
2464     }
2465     (*kc->dec)(kc);
2466     return res;
2467 }
2468
2469 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2470                                oid_value attributeSet,
2471                                NMEM stream, NMEM rset_nmem,
2472                                Z_SortKeySpecList *sort_sequence,
2473                                int num_bases, char **basenames,
2474                                RSET **result_sets, int *num_result_sets,
2475                                Z_Operator *parent_op,
2476                                struct rset_key_control *kc)
2477 {
2478     *num_result_sets = 0;
2479     if (zs->which == Z_RPNStructure_complex)
2480     {
2481         ZEBRA_RES res;
2482         Z_Operator *zop = zs->u.complex->roperator;
2483         RSET *result_sets_l = 0;
2484         int num_result_sets_l = 0;
2485         RSET *result_sets_r = 0;
2486         int num_result_sets_r = 0;
2487
2488         res = rpn_search_structure(zh, zs->u.complex->s1,
2489                                    attributeSet, stream, rset_nmem,
2490                                    sort_sequence,
2491                                    num_bases, basenames,
2492                                    &result_sets_l, &num_result_sets_l,
2493                                    zop, kc);
2494         if (res != ZEBRA_OK)
2495         {
2496             int i;
2497             for (i = 0; i<num_result_sets_l; i++)
2498                 rset_delete(result_sets_l[i]);
2499             return res;
2500         }
2501         res = rpn_search_structure(zh, zs->u.complex->s2,
2502                                    attributeSet, stream, rset_nmem,
2503                                    sort_sequence,
2504                                    num_bases, basenames,
2505                                    &result_sets_r, &num_result_sets_r,
2506                                    zop, kc);
2507         if (res != ZEBRA_OK)
2508         {
2509             int i;
2510             for (i = 0; i<num_result_sets_l; i++)
2511                 rset_delete(result_sets_l[i]);
2512             for (i = 0; i<num_result_sets_r; i++)
2513                 rset_delete(result_sets_r[i]);
2514             return res;
2515         }
2516
2517         /* make a new list of result for all children */
2518         *num_result_sets = num_result_sets_l + num_result_sets_r;
2519         *result_sets = nmem_malloc(stream, *num_result_sets *
2520                                    sizeof(**result_sets));
2521         memcpy(*result_sets, result_sets_l,
2522                num_result_sets_l * sizeof(**result_sets));
2523         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2524                num_result_sets_r * sizeof(**result_sets));
2525
2526         if (!parent_op || parent_op->which != zop->which
2527             || (zop->which != Z_Operator_and &&
2528                 zop->which != Z_Operator_or))
2529         {
2530             /* parent node different from this one (or non-present) */
2531             /* we must combine result sets now */
2532             RSET rset;
2533             switch (zop->which)
2534             {
2535             case Z_Operator_and:
2536                 rset = rsmulti_and_create(rset_nmem, kc,
2537                                           kc->scope,
2538                                           *num_result_sets, *result_sets);
2539                 break;
2540             case Z_Operator_or:
2541                 rset = rsmulti_or_create(rset_nmem, kc,
2542                                          kc->scope, 0, /* termid */
2543                                          *num_result_sets, *result_sets);
2544                 break;
2545             case Z_Operator_and_not:
2546                 rset = rsbool_create_not(rset_nmem, kc,
2547                                          kc->scope,
2548                                          (*result_sets)[0],
2549                                          (*result_sets)[1]);
2550                 break;
2551             case Z_Operator_prox:
2552                 if (zop->u.prox->which != Z_ProximityOperator_known)
2553                 {
2554                     zebra_setError(zh,
2555                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2556                                    0);
2557                     return ZEBRA_FAIL;
2558                 }
2559                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2560                 {
2561                     zebra_setError_zint(zh,
2562                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2563                                         *zop->u.prox->u.known);
2564                     return ZEBRA_FAIL;
2565                 }
2566                 else
2567                 {
2568                     rset = rsprox_create(rset_nmem, kc,
2569                                          kc->scope,
2570                                          *num_result_sets, *result_sets,
2571                                          *zop->u.prox->ordered,
2572                                          (!zop->u.prox->exclusion ?
2573                                           0 : *zop->u.prox->exclusion),
2574                                          *zop->u.prox->relationType,
2575                                          *zop->u.prox->distance );
2576                 }
2577                 break;
2578             default:
2579                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2580                 return ZEBRA_FAIL;
2581             }
2582             *num_result_sets = 1;
2583             *result_sets = nmem_malloc(stream, *num_result_sets *
2584                                        sizeof(**result_sets));
2585             (*result_sets)[0] = rset;
2586         }
2587     }
2588     else if (zs->which == Z_RPNStructure_simple)
2589     {
2590         RSET rset;
2591         ZEBRA_RES res;
2592
2593         if (zs->u.simple->which == Z_Operand_APT)
2594         {
2595             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2596             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2597                                  attributeSet, stream, sort_sequence,
2598                                  num_bases, basenames, rset_nmem, &rset,
2599                                  kc);
2600             if (res != ZEBRA_OK)
2601                 return res;
2602         }
2603         else if (zs->u.simple->which == Z_Operand_resultSetId)
2604         {
2605             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2606             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2607             if (!rset)
2608             {
2609                 zebra_setError(zh,
2610                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2611                                zs->u.simple->u.resultSetId);
2612                 return ZEBRA_FAIL;
2613             }
2614             rset_dup(rset);
2615         }
2616         else
2617         {
2618             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2619             return ZEBRA_FAIL;
2620         }
2621         *num_result_sets = 1;
2622         *result_sets = nmem_malloc(stream, *num_result_sets *
2623                                    sizeof(**result_sets));
2624         (*result_sets)[0] = rset;
2625     }
2626     else
2627     {
2628         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2629         return ZEBRA_FAIL;
2630     }
2631     return ZEBRA_OK;
2632 }
2633
2634 struct scan_info_entry {
2635     char *term;
2636     ISAM_P isam_p;
2637 };
2638
2639 struct scan_info {
2640     struct scan_info_entry *list;
2641     ODR odr;
2642     int before, after;
2643     char prefix[20];
2644 };
2645
2646 static int scan_handle (char *name, const char *info, int pos, void *client)
2647 {
2648     int len_prefix, idx;
2649     struct scan_info *scan_info = (struct scan_info *) client;
2650
2651     len_prefix = strlen(scan_info->prefix);
2652     if (memcmp (name, scan_info->prefix, len_prefix))
2653         return 1;
2654     if (pos > 0)
2655         idx = scan_info->after - pos + scan_info->before;
2656     else
2657         idx = - pos - 1;
2658
2659     if (idx < 0)
2660         return 0;
2661     scan_info->list[idx].term = (char *)
2662         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2663     strcpy(scan_info->list[idx].term, name + len_prefix);
2664     assert (*info == sizeof(ISAM_P));
2665     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2666     return 0;
2667 }
2668
2669 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2670                               char **dst, const char *src)
2671 {
2672     char term_src[IT_MAX_WORD];
2673     char term_dst[IT_MAX_WORD];
2674
2675     zebra_term_untrans (zh, reg_type, term_src, src);
2676
2677     if (zh->iconv_from_utf8 != 0)
2678     {
2679         int len;
2680         char *inbuf = term_src;
2681         size_t inleft = strlen(term_src);
2682         char *outbuf = term_dst;
2683         size_t outleft = sizeof(term_dst)-1;
2684         size_t ret;
2685
2686         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2687                          &outbuf, &outleft);
2688         if (ret == (size_t)(-1))
2689             len = 0;
2690         else
2691             len = outbuf - term_dst;
2692         *dst = nmem_malloc(stream, len + 1);
2693         if (len > 0)
2694             memcpy (*dst, term_dst, len);
2695         (*dst)[len] = '\0';
2696     }
2697     else
2698         *dst = nmem_strdup(stream, term_src);
2699 }
2700
2701 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2702 {
2703     zint psysno = 0;
2704     struct it_key key;
2705     RSFD rfd;
2706
2707     yaz_log(YLOG_DEBUG, "count_set");
2708
2709     rset->hits_limit = zh->approx_limit;
2710
2711     *count = 0;
2712     rfd = rset_open(rset, RSETF_READ);
2713     while (rset_read(rfd, &key,0 /* never mind terms */))
2714     {
2715         if (key.mem[0] != psysno)
2716         {
2717             psysno = key.mem[0];
2718             if (rfd->counted_items >= rset->hits_limit)
2719                 break;
2720         }
2721     }
2722     rset_close (rfd);
2723     *count = rset->hits_count;
2724 }
2725
2726 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2727                    oid_value attributeset,
2728                    int num_bases, char **basenames,
2729                    int *position, int *num_entries, ZebraScanEntry **list,
2730                    int *is_partial, RSET limit_set, int return_zero)
2731 {
2732     int i;
2733     int pos = *position;
2734     int num = *num_entries;
2735     int before;
2736     int after;
2737     int base_no;
2738     char termz[IT_MAX_WORD+20];
2739     AttrType use;
2740     int use_value;
2741     const char *use_string = 0;
2742     struct scan_info *scan_info_array;
2743     ZebraScanEntry *glist;
2744     int ords[32], ord_no = 0;
2745     int ptr[32];
2746
2747     int bases_ok = 0;     /* no of databases with OK attribute */
2748     int errCode = 0;      /* err code (if any is not OK) */
2749     char *errString = 0;  /* addinfo */
2750
2751     unsigned reg_id;
2752     char *search_type = NULL;
2753     char rank_type[128];
2754     int complete_flag;
2755     int sort_flag;
2756     NMEM rset_nmem = NULL;
2757     struct rset_key_control *kc = 0;
2758
2759     *list = 0;
2760     *is_partial = 0;
2761
2762     if (attributeset == VAL_NONE)
2763         attributeset = VAL_BIB1;
2764
2765     if (!limit_set)
2766     {
2767         AttrType termset;
2768         int termset_value_numeric;
2769         const char *termset_value_string;
2770         attr_init(&termset, zapt, 8);
2771         termset_value_numeric =
2772             attr_find_ex(&termset, NULL, &termset_value_string);
2773         if (termset_value_numeric != -1)
2774         {
2775             char resname[32];
2776             const char *termset_name = 0;
2777
2778             if (termset_value_numeric != -2)
2779             {
2780
2781                 sprintf(resname, "%d", termset_value_numeric);
2782                 termset_name = resname;
2783             }
2784             else
2785                 termset_name = termset_value_string;
2786
2787             limit_set = resultSetRef (zh, termset_name);
2788         }
2789     }
2790
2791     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2792             pos, num, attributeset);
2793
2794     attr_init(&use, zapt, 1);
2795     use_value = attr_find_ex(&use, &attributeset, &use_string);
2796
2797     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2798                         rank_type, &complete_flag, &sort_flag))
2799     {
2800         *num_entries = 0;
2801         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2802         return ZEBRA_FAIL;
2803     }
2804     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2805
2806     if (use_value == -1)
2807         use_value = 1016;
2808     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2809     {
2810         data1_local_attribute *local_attr;
2811         attent attp;
2812         int ord;
2813
2814         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2815         {
2816             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2817                            basenames[base_no]);
2818             *num_entries = 0;
2819             return ZEBRA_FAIL;
2820         }
2821
2822         if (use_string &&
2823             (ord = zebraExplain_lookup_attr_str(zh->reg->zei, reg_id,
2824                                                 use_string)) >= 0)
2825         {
2826             /* we have a match for a raw string attribute */
2827             if (ord > 0)
2828                 ords[ord_no++] = ord;
2829             attp.local_attributes = 0;  /* no more attributes */
2830         }
2831         else
2832         {
2833             int r;
2834
2835             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2836                                       use_string)))
2837             {
2838                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2839                         attributeset, use_value);
2840                 if (r == -1)
2841                 {
2842                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2843                     if (use_string)
2844                         zebra_setError(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
2845                                        use_string);
2846                     else
2847                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
2848                                             use_value);
2849                 }
2850                 else
2851                 {
2852                     zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_SET, 0);
2853                 }
2854                 continue;
2855             }
2856         }
2857         bases_ok++;
2858         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2859              local_attr = local_attr->next)
2860         {
2861             ord = zebraExplain_lookup_attr_su(zh->reg->zei, reg_id,
2862                                               attp.attset_ordinal,
2863                                               local_attr->local);
2864             if (ord > 0)
2865                 ords[ord_no++] = ord;
2866         }
2867     }
2868     if (!bases_ok && errCode)
2869     {
2870         zebra_setError(zh, errCode, errString);
2871         *num_entries = 0;
2872         return ZEBRA_FAIL;
2873     }
2874     if (ord_no == 0)
2875     {
2876         *num_entries = 0;
2877         return ZEBRA_OK;
2878     }
2879     /* prepare dictionary scanning */
2880     if (num < 1)
2881     {
2882         *num_entries = 0;
2883         return ZEBRA_OK;
2884     }
2885     before = pos-1;
2886     if (before < 0)
2887         before = 0;
2888     after = 1+num-pos;
2889     if (after < 0)
2890         after = 0;
2891     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2892             "after=%d before+after=%d",
2893             pos, num, before, after, before+after);
2894     scan_info_array = (struct scan_info *)
2895         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2896     for (i = 0; i < ord_no; i++)
2897     {
2898         int j, prefix_len = 0;
2899         int before_tmp = before, after_tmp = after;
2900         struct scan_info *scan_info = scan_info_array + i;
2901         struct rpn_char_map_info rcmi;
2902
2903         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2904
2905         scan_info->before = before;
2906         scan_info->after = after;
2907         scan_info->odr = stream;
2908
2909         scan_info->list = (struct scan_info_entry *)
2910             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2911         for (j = 0; j<before+after; j++)
2912             scan_info->list[j].term = NULL;
2913
2914         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2915         termz[prefix_len] = 0;
2916         strcpy(scan_info->prefix, termz);
2917
2918         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2919             return ZEBRA_FAIL;
2920
2921         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2922                   scan_info, scan_handle);
2923     }
2924     glist = (ZebraScanEntry *)
2925         odr_malloc(stream, (before+after)*sizeof(*glist));
2926
2927     rset_nmem = nmem_create();
2928     kc = zebra_key_control_create(zh);
2929
2930     /* consider terms after main term */
2931     for (i = 0; i < ord_no; i++)
2932         ptr[i] = before;
2933
2934     *is_partial = 0;
2935     for (i = 0; i<after; i++)
2936     {
2937         int j, j0 = -1;
2938         const char *mterm = NULL;
2939         const char *tst;
2940         RSET rset = 0;
2941         int lo = i + pos-1; /* offset in result list */
2942
2943         /* find: j0 is the first of the minimal values */
2944         for (j = 0; j < ord_no; j++)
2945         {
2946             if (ptr[j] < before+after && ptr[j] >= 0 &&
2947                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2948                 (!mterm || strcmp (tst, mterm) < 0))
2949             {
2950                 j0 = j;
2951                 mterm = tst;
2952             }
2953         }
2954         if (j0 == -1)
2955             break;  /* no value found, stop */
2956
2957         /* get result set for first one , but only if it's within bounds */
2958         if (lo >= 0)
2959         {
2960             /* get result set for first term */
2961             zebra_term_untrans_iconv(zh, stream->mem, reg_id,
2962                                      &glist[lo].term, mterm);
2963             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2964                               glist[lo].term, strlen(glist[lo].term),
2965                               NULL, 0, zapt->term->which, rset_nmem,
2966                               kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
2967                               0 /* term_ref_id_str */);
2968         }
2969         ptr[j0]++; /* move index for this set .. */
2970         /* get result set for remaining scan terms */
2971         for (j = j0+1; j<ord_no; j++)
2972         {
2973             if (ptr[j] < before+after && ptr[j] >= 0 &&
2974                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2975                 !strcmp (tst, mterm))
2976             {
2977                 if (lo >= 0)
2978                 {
2979                     RSET rsets[2];
2980
2981                     rsets[0] = rset;
2982                     rsets[1] =
2983                         rset_trunc(
2984                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2985                             glist[lo].term,
2986                             strlen(glist[lo].term), NULL, 0,
2987                             zapt->term->which,rset_nmem,
2988                             kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
2989                             0 /* term_ref_id_str */ );
2990                     rset = rsmulti_or_create(rset_nmem, kc,
2991                                              kc->scope, 0 /* termid */,
2992                                              2, rsets);
2993                 }
2994                 ptr[j]++;
2995             }
2996         }
2997         if (lo >= 0)
2998         {
2999             zint count;
3000             /* merge with limit_set if given */
3001             if (limit_set)
3002             {
3003                 RSET rsets[2];
3004                 rsets[0] = rset;
3005                 rsets[1] = rset_dup(limit_set);
3006
3007                 rset = rsmulti_and_create(rset_nmem, kc,
3008                                           kc->scope,
3009                                           2, rsets);
3010             }
3011             /* count it */
3012             count_set(zh, rset, &count);
3013             glist[lo].occurrences = count;
3014             rset_delete(rset);
3015         }
3016     }
3017     if (i < after)
3018     {
3019         *num_entries -= (after-i);
3020         *is_partial = 1;
3021         if (*num_entries < 0)
3022         {
3023             (*kc->dec)(kc);
3024             nmem_destroy(rset_nmem);
3025             *num_entries = 0;
3026             return ZEBRA_OK;
3027         }
3028     }
3029     /* consider terms before main term */
3030     for (i = 0; i<ord_no; i++)
3031         ptr[i] = 0;
3032
3033     for (i = 0; i<before; i++)
3034     {
3035         int j, j0 = -1;
3036         const char *mterm = NULL;
3037         const char *tst;
3038         RSET rset;
3039         int lo = before-1-i; /* offset in result list */
3040         zint count;
3041
3042         for (j = 0; j <ord_no; j++)
3043         {
3044             if (ptr[j] < before && ptr[j] >= 0 &&
3045                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3046                 (!mterm || strcmp (tst, mterm) > 0))
3047             {
3048                 j0 = j;
3049                     mterm = tst;
3050             }
3051         }
3052         if (j0 == -1)
3053             break;
3054
3055         zebra_term_untrans_iconv(zh, stream->mem, reg_id,
3056                                  &glist[lo].term, mterm);
3057
3058         rset = rset_trunc
3059             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
3060              glist[lo].term, strlen(glist[lo].term),
3061              NULL, 0, zapt->term->which, rset_nmem,
3062              kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3063              0 /* term_ref_id_str */);
3064
3065         ptr[j0]++;
3066
3067         for (j = j0+1; j<ord_no; j++)
3068         {
3069             if (ptr[j] < before && ptr[j] >= 0 &&
3070                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3071                 !strcmp (tst, mterm))
3072             {
3073                 RSET rsets[2];
3074
3075                 rsets[0] = rset;
3076                 rsets[1] = rset_trunc(
3077                     zh,
3078                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
3079                     glist[lo].term,
3080                     strlen(glist[lo].term), NULL, 0,
3081                     zapt->term->which, rset_nmem,
3082                     kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3083                     0 /* term_ref_id_str */);
3084                 rset = rsmulti_or_create(rset_nmem, kc,
3085                                          kc->scope, 0 /* termid */, 2, rsets);
3086
3087                 ptr[j]++;
3088             }
3089         }
3090         if (limit_set)
3091         {
3092             RSET rsets[2];
3093             rsets[0] = rset;
3094             rsets[1] = rset_dup(limit_set);
3095
3096             rset = rsmulti_and_create(rset_nmem, kc,
3097                                       kc->scope, 2, rsets);
3098         }
3099         count_set(zh, rset, &count);
3100         glist[lo].occurrences = count;
3101         rset_delete (rset);
3102     }
3103     (*kc->dec)(kc);
3104     nmem_destroy(rset_nmem);
3105     i = before-i;
3106     if (i)
3107     {
3108         *is_partial = 1;
3109         *position -= i;
3110         *num_entries -= i;
3111         if (*num_entries <= 0)
3112         {
3113             *num_entries = 0;
3114             return ZEBRA_OK;
3115         }
3116     }
3117
3118     *list = glist + i;               /* list is set to first 'real' entry */
3119
3120     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3121             *position, *num_entries);
3122     return ZEBRA_OK;
3123 }
3124