index/zrpn.c

   1 /* $Id: zrpn.c,v 1.168 2005-01-15 20:47:15 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39 /* maximum number of terms in an and/or/phrase item */
  40 #define TERM_LIST_LENGTH_MAX 256
  41
  42 static const struct key_control it_ctrl =
  43 {
  44     sizeof(struct it_key),
  45     2, /* we have sysnos and seqnos in this key, nothing more */
  46     key_compare_it,
  47     key_logdump_txt,   /* FIXME  - clean up these functions */
  48     key_get_seq,
  49 };
  50
  51
  52 const struct key_control *key_it_ctrl = &it_ctrl;
  53
  54 struct rpn_char_map_info
  55 {
  56     ZebraMaps zm;
  57     int reg_type;
  58 };
  59
  60 typedef struct
  61 {
  62     int type;
  63     int major;
  64     int minor;
  65     Z_AttributesPlusTerm *zapt;
  66 } AttrType;
  67
  68
  69 static int log_level_set = 0;
  70 static int log_level_rpn = 0;
  71
  72 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  73 {
  74     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  75     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  76 #if 0
  77     if (out && *out)
  78     {
  79         const char *outp = *out;
  80         yaz_log(YLOG_LOG, "---");
  81         while (*outp)
  82         {
  83             yaz_log(YLOG_LOG, "%02X", *outp);
  84             outp++;
  85         }
  86     }
  87 #endif
  88     return out;
  89 }
  90
  91 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  92                                   struct rpn_char_map_info *map_info)
  93 {
  94     map_info->zm = reg->zebra_maps;
  95     map_info->reg_type = reg_type;
  96     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  97 }
  98
  99 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
 100                          const char **string_value)
 101 {
 102     int num_attributes;
 103
 104     num_attributes = src->zapt->attributes->num_attributes;
 105     while (src->major < num_attributes)
 106     {
 107         Z_AttributeElement *element;
 108
 109         element = src->zapt->attributes->attributes[src->major];
 110         if (src->type == *element->attributeType)
 111         {
 112             switch (element->which)
 113             {
 114             case Z_AttributeValue_numeric:
 115                 ++(src->major);
 116                 if (element->attributeSet && attributeSetP)
 117                 {
 118                     oident *attrset;
 119
 120                     attrset = oid_getentbyoid(element->attributeSet);
 121                     *attributeSetP = attrset->value;
 122                 }
 123                 return *element->value.numeric;
 124                 break;
 125             case Z_AttributeValue_complex:
 126                 if (src->minor >= element->value.complex->num_list)
 127                     break;
 128                 if (element->attributeSet && attributeSetP)
 129                 {
 130                     oident *attrset;
 131
 132                     attrset = oid_getentbyoid(element->attributeSet);
 133                     *attributeSetP = attrset->value;
 134                 }
 135                 if (element->value.complex->list[src->minor]->which ==
 136                     Z_StringOrNumeric_numeric)
 137                 {
 138                     ++(src->minor);
 139                     return
 140                         *element->value.complex->list[src->minor-1]->u.numeric;
 141                 }
 142                 else if (element->value.complex->list[src->minor]->which ==
 143                          Z_StringOrNumeric_string)
 144                 {
 145                     if (!string_value)
 146                         break;
 147                     ++(src->minor);
 148                     *string_value =
 149                         element->value.complex->list[src->minor-1]->u.string;
 150                     return -2;
 151                 }
 152                 else
 153                     break;
 154             default:
 155                 assert(0);
 156             }
 157         }
 158         ++(src->major);
 159     }
 160     return -1;
 161 }
 162
 163 static int attr_find(AttrType *src, oid_value *attributeSetP)
 164 {
 165     return attr_find_ex(src, attributeSetP, 0);
 166 }
 167
 168 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 169                        int type)
 170 {
 171     src->zapt = zapt;
 172     src->type = type;
 173     src->major = 0;
 174     src->minor = 0;
 175 }
 176
 177 #define TERM_COUNT
 178
 179 struct grep_info {
 180 #ifdef TERM_COUNT
 181     int *term_no;
 182 #endif
 183     ISAMC_P *isam_p_buf;
 184     int isam_p_size;
 185     int isam_p_indx;
 186     ZebraHandle zh;
 187     int reg_type;
 188     ZebraSet termset;
 189 };
 190
 191 static void term_untrans(ZebraHandle zh, int reg_type,
 192                            char *dst, const char *src)
 193 {
 194     int len = 0;
 195     while (*src)
 196     {
 197         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 198                                            reg_type, &src);
 199         if (!cp && len < IT_MAX_WORD-1)
 200             dst[len++] = *src++;
 201         else
 202             while (*cp && len < IT_MAX_WORD-1)
 203                 dst[len++] = *cp++;
 204     }
 205     dst[len] = '\0';
 206 }
 207
 208 static void add_isam_p(const char *name, const char *info,
 209                         struct grep_info *p)
 210 {
 211     if (!log_level_set)
 212     {
 213         log_level_rpn = yaz_log_module_level("rpn");
 214         log_level_set = 1;
 215     }
 216     if (p->isam_p_indx == p->isam_p_size)
 217     {
 218         ISAMC_P *new_isam_p_buf;
 219 #ifdef TERM_COUNT
 220         int *new_term_no;
 221 #endif
 222         p->isam_p_size = 2*p->isam_p_size + 100;
 223         new_isam_p_buf = (ISAMC_P *) xmalloc(sizeof(*new_isam_p_buf) *
 224                                              p->isam_p_size);
 225         if (p->isam_p_buf)
 226         {
 227             memcpy(new_isam_p_buf, p->isam_p_buf,
 228                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 229             xfree(p->isam_p_buf);
 230         }
 231         p->isam_p_buf = new_isam_p_buf;
 232
 233 #ifdef TERM_COUNT
 234         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 235         if (p->term_no)
 236         {
 237             memcpy(new_term_no, p->isam_p_buf,
 238                     p->isam_p_indx * sizeof(*p->term_no));
 239             xfree(p->term_no);
 240         }
 241         p->term_no = new_term_no;
 242 #endif
 243     }
 244     assert(*info == sizeof(*p->isam_p_buf));
 245     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 246
 247 #if 1
 248     if (p->termset)
 249     {
 250         const char *db;
 251         int set, use;
 252         char term_tmp[IT_MAX_WORD];
 253         int su_code = 0;
 254         int len = key_SU_decode (&su_code, name);
 255
 256         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 257         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 258         zebraExplain_lookup_ord (p->zh->reg->zei,
 259                                  su_code, &db, &set, &use);
 260         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 261
 262         resultSetAddTerm(p->zh, p->termset, name[len], db,
 263                          set, use, term_tmp);
 264     }
 265 #endif
 266     (p->isam_p_indx)++;
 267 }
 268
 269 static int grep_handle(char *name, const char *info, void *p)
 270 {
 271     add_isam_p(name, info, (struct grep_info *) p);
 272     return 0;
 273 }
 274
 275 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 276                      const char *ct1, const char *ct2, int first)
 277 {
 278     const char *s1, *s0 = *src;
 279     const char **map;
 280
 281     /* skip white space */
 282     while (*s0)
 283     {
 284         if (ct1 && strchr(ct1, *s0))
 285             break;
 286         if (ct2 && strchr(ct2, *s0))
 287             break;
 288         s1 = s0;
 289         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 290         if (**map != *CHR_SPACE)
 291             break;
 292         s0 = s1;
 293     }
 294     *src = s0;
 295     return *s0;
 296 }
 297
 298 #define REGEX_CHARS " []()|.*+?!"
 299
 300 /* term_100: handle term, where trunc = none(no operators at all) */
 301 static int term_100(ZebraMaps zebra_maps, int reg_type,
 302                      const char **src, char *dst, int space_split,
 303                      char *dst_term)
 304 {
 305     const char *s0, *s1;
 306     const char **map;
 307     int i = 0;
 308     int j = 0;
 309
 310     const char *space_start = 0;
 311     const char *space_end = 0;
 312
 313     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 314         return 0;
 315     s0 = *src;
 316     while (*s0)
 317     {
 318         s1 = s0;
 319         map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 320         if (space_split)
 321         {
 322             if (**map == *CHR_SPACE)
 323                 break;
 324         }
 325         else  /* complete subfield only. */
 326         {
 327             if (**map == *CHR_SPACE)
 328             {   /* save space mapping for later  .. */
 329                 space_start = s1;
 330                 space_end = s0;
 331                 continue;
 332             }
 333             else if (space_start)
 334             {   /* reload last space */
 335                 while (space_start < space_end)
 336                 {
 337                     if (strchr(REGEX_CHARS, *space_start))
 338                         dst[i++] = '\\';
 339                     dst_term[j++] = *space_start;
 340                     dst[i++] = *space_start++;
 341                 }
 342                 /* and reset */
 343                 space_start = space_end = 0;
 344             }
 345         }
 346         /* add non-space char */
 347         while (s1 < s0)
 348         {
 349             if (strchr(REGEX_CHARS, *s1))
 350                 dst[i++] = '\\';
 351             dst_term[j++] = *s1;
 352             dst[i++] = *s1++;
 353         }
 354     }
 355     dst[i] = '\0';
 356     dst_term[j] = '\0';
 357     *src = s0;
 358     return i;
 359 }
 360
 361 /* term_101: handle term, where trunc = Process # */
 362 static int term_101(ZebraMaps zebra_maps, int reg_type,
 363                      const char **src, char *dst, int space_split,
 364                      char *dst_term)
 365 {
 366     const char *s0, *s1;
 367     const char **map;
 368     int i = 0;
 369     int j = 0;
 370
 371     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 372         return 0;
 373     s0 = *src;
 374     while (*s0)
 375     {
 376         if (*s0 == '#')
 377         {
 378             dst[i++] = '.';
 379             dst[i++] = '*';
 380             dst_term[j++] = *s0++;
 381         }
 382         else
 383         {
 384             s1 = s0;
 385             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 386             if (space_split && **map == *CHR_SPACE)
 387                 break;
 388             while (s1 < s0)
 389             {
 390                 if (strchr(REGEX_CHARS, *s1))
 391                     dst[i++] = '\\';
 392                 dst_term[j++] = *s1;
 393                 dst[i++] = *s1++;
 394             }
 395         }
 396     }
 397     dst[i] = '\0';
 398     dst_term[j++] = '\0';
 399     *src = s0;
 400     return i;
 401 }
 402
 403 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 404 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 405                      char *dst, int *errors, int space_split,
 406                      char *dst_term)
 407 {
 408     int i = 0;
 409     int j = 0;
 410     const char *s0, *s1;
 411     const char **map;
 412
 413     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 414         return 0;
 415     s0 = *src;
 416     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 417         isdigit(((const unsigned char *)s0)[1]))
 418     {
 419         *errors = s0[1] - '0';
 420         s0 += 3;
 421         if (*errors > 3)
 422             *errors = 3;
 423     }
 424     while (*s0)
 425     {
 426         if (strchr("^\\()[].*+?|-", *s0))
 427         {
 428             dst_term[j++] = *s0;
 429             dst[i++] = *s0++;
 430         }
 431         else
 432         {
 433             s1 = s0;
 434             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 435             if (**map == *CHR_SPACE)
 436                 break;
 437             while (s1 < s0)
 438             {
 439                 if (strchr(REGEX_CHARS, *s1))
 440                     dst[i++] = '\\';
 441                 dst_term[j++] = *s1;
 442                 dst[i++] = *s1++;
 443             }
 444         }
 445     }
 446     dst[i] = '\0';
 447     dst_term[j] = '\0';
 448     *src = s0;
 449     return i;
 450 }
 451
 452 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 453 static int term_102 (ZebraMaps zebra_maps, int reg_type, const char **src,
 454                      char *dst, int space_split, char *dst_term)
 455 {
 456     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 457                      dst_term);
 458 }
 459
 460
 461 /* term_104: handle term, where trunc = Process # and ! */
 462 static int term_104(ZebraMaps zebra_maps, int reg_type,
 463                      const char **src, char *dst, int space_split,
 464                      char *dst_term)
 465 {
 466     const char *s0, *s1;
 467     const char **map;
 468     int i = 0;
 469     int j = 0;
 470
 471     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 472         return 0;
 473     s0 = *src;
 474     while (*s0)
 475     {
 476         if (*s0 == '?')
 477         {
 478             dst_term[j++] = *s0++;
 479             if (*s0 >= '0' && *s0 <= '9')
 480             {
 481                 int limit = 0;
 482                 while (*s0 >= '0' && *s0 <= '9')
 483                 {
 484                     limit = limit * 10 + (*s0 - '0');
 485                     dst_term[j++] = *s0++;
 486                 }
 487                 if (limit > 20)
 488                     limit = 20;
 489                 while (--limit >= 0)
 490                 {
 491                     dst[i++] = '.';
 492                     dst[i++] = '?';
 493                 }
 494             }
 495             else
 496             {
 497                 dst[i++] = '.';
 498                 dst[i++] = '*';
 499             }
 500         }
 501         else if (*s0 == '*')
 502         {
 503             dst[i++] = '.';
 504             dst[i++] = '*';
 505             dst_term[j++] = *s0++;
 506         }
 507         else if (*s0 == '#')
 508         {
 509             dst[i++] = '.';
 510             dst_term[j++] = *s0++;
 511         }
 512         {
 513             s1 = s0;
 514             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 515             if (space_split && **map == *CHR_SPACE)
 516                 break;
 517             while (s1 < s0)
 518             {
 519                 if (strchr(REGEX_CHARS, *s1))
 520                     dst[i++] = '\\';
 521                 dst_term[j++] = *s1;
 522                 dst[i++] = *s1++;
 523             }
 524         }
 525     }
 526     dst[i] = '\0';
 527     dst_term[j++] = '\0';
 528     *src = s0;
 529     return i;
 530 }
 531
 532 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 533 static int term_105 (ZebraMaps zebra_maps, int reg_type,
 534                      const char **src, char *dst, int space_split,
 535                      char *dst_term, int right_truncate)
 536 {
 537     const char *s0, *s1;
 538     const char **map;
 539     int i = 0;
 540     int j = 0;
 541
 542     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 543         return 0;
 544     s0 = *src;
 545     while (*s0)
 546     {
 547         if (*s0 == '*')
 548         {
 549             dst[i++] = '.';
 550             dst[i++] = '*';
 551             dst_term[j++] = *s0++;
 552         }
 553         else if (*s0 == '!')
 554         {
 555             dst[i++] = '.';
 556             dst_term[j++] = *s0++;
 557         }
 558         {
 559             s1 = s0;
 560             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 561             if (space_split && **map == *CHR_SPACE)
 562                 break;
 563             while (s1 < s0)
 564             {
 565                 if (strchr(REGEX_CHARS, *s1))
 566                     dst[i++] = '\\';
 567                 dst_term[j++] = *s1;
 568                 dst[i++] = *s1++;
 569             }
 570         }
 571     }
 572     if (right_truncate)
 573     {
 574         dst[i++] = '.';
 575         dst[i++] = '*';
 576     }
 577     dst[i] = '\0';
 578
 579     dst_term[j++] = '\0';
 580     *src = s0;
 581     return i;
 582 }
 583
 584
 585 /* gen_regular_rel - generate regular expression from relation
 586  *  val:     border value (inclusive)
 587  *  islt:    1 if <=; 0 if >=.
 588  */
 589 static void gen_regular_rel(char *dst, int val, int islt)
 590 {
 591     int dst_p;
 592     int w, d, i;
 593     int pos = 0;
 594     char numstr[20];
 595
 596     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 597     if (val >= 0)
 598     {
 599         if (islt)
 600             strcpy(dst, "(-[0-9]+|(");
 601         else
 602             strcpy(dst, "((");
 603     }
 604     else
 605     {
 606         if (!islt)
 607         {
 608             strcpy(dst, "([0-9]+|-(");
 609             dst_p = strlen(dst);
 610             islt = 1;
 611         }
 612         else
 613         {
 614             strcpy(dst, "(-(");
 615             islt = 0;
 616         }
 617         val = -val;
 618     }
 619     dst_p = strlen(dst);
 620     sprintf(numstr, "%d", val);
 621     for (w = strlen(numstr); --w >= 0; pos++)
 622     {
 623         d = numstr[w];
 624         if (pos > 0)
 625         {
 626             if (islt)
 627             {
 628                 if (d == '0')
 629                     continue;
 630                 d--;
 631             }
 632             else
 633             {
 634                 if (d == '9')
 635                     continue;
 636                 d++;
 637             }
 638         }
 639
 640         strcpy(dst + dst_p, numstr);
 641         dst_p = strlen(dst) - pos - 1;
 642
 643         if (islt)
 644         {
 645             if (d != '0')
 646             {
 647                 dst[dst_p++] = '[';
 648                 dst[dst_p++] = '0';
 649                 dst[dst_p++] = '-';
 650                 dst[dst_p++] = d;
 651                 dst[dst_p++] = ']';
 652             }
 653             else
 654                 dst[dst_p++] = d;
 655         }
 656         else
 657         {
 658             if (d != '9')
 659             {
 660                 dst[dst_p++] = '[';
 661                 dst[dst_p++] = d;
 662                 dst[dst_p++] = '-';
 663                 dst[dst_p++] = '9';
 664                 dst[dst_p++] = ']';
 665             }
 666             else
 667                 dst[dst_p++] = d;
 668         }
 669         for (i = 0; i<pos; i++)
 670         {
 671             dst[dst_p++] = '[';
 672             dst[dst_p++] = '0';
 673             dst[dst_p++] = '-';
 674             dst[dst_p++] = '9';
 675             dst[dst_p++] = ']';
 676         }
 677         dst[dst_p++] = '|';
 678     }
 679     dst[dst_p] = '\0';
 680     if (islt)
 681     {
 682         /* match everything less than 10^(pos-1) */
 683         strcat(dst, "0*");
 684         for (i = 1; i<pos; i++)
 685             strcat(dst, "[0-9]?");
 686     }
 687     else
 688     {
 689         /* match everything greater than 10^pos */
 690         for (i = 0; i <= pos; i++)
 691             strcat(dst, "[0-9]");
 692         strcat(dst, "[0-9]*");
 693     }
 694     strcat(dst, "))");
 695 }
 696
 697 void string_rel_add_char(char **term_p, const char *src, int *indx)
 698 {
 699     if (src[*indx] == '\\')
 700         *(*term_p)++ = src[(*indx)++];
 701     *(*term_p)++ = src[(*indx)++];
 702 }
 703
 704 /*
 705  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 706  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 707  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 708  *              ([^-a].*|a[^-b].*|ab[c-].*)
 709  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 710  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 711  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 712  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 713  */
 714 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 715                             const char **term_sub, char *term_dict,
 716                             oid_value attributeSet,
 717                             int reg_type, int space_split, char *term_dst)
 718 {
 719     AttrType relation;
 720     int relation_value;
 721     int i;
 722     char *term_tmp = term_dict + strlen(term_dict);
 723     char term_component[2*IT_MAX_WORD+20];
 724
 725     attr_init(&relation, zapt, 2);
 726     relation_value = attr_find(&relation, NULL);
 727
 728     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 729     switch (relation_value)
 730     {
 731     case 1:
 732         if (!term_100 (zh->reg->zebra_maps, reg_type,
 733                        term_sub, term_component,
 734                        space_split, term_dst))
 735             return 0;
 736         yaz_log(log_level_rpn, "Relation <");
 737
 738         *term_tmp++ = '(';
 739         for (i = 0; term_component[i]; )
 740         {
 741             int j = 0;
 742
 743             if (i)
 744                 *term_tmp++ = '|';
 745             while (j < i)
 746                 string_rel_add_char (&term_tmp, term_component, &j);
 747
 748             *term_tmp++ = '[';
 749
 750             *term_tmp++ = '^';
 751             string_rel_add_char (&term_tmp, term_component, &i);
 752             *term_tmp++ = '-';
 753
 754             *term_tmp++ = ']';
 755             *term_tmp++ = '.';
 756             *term_tmp++ = '*';
 757
 758             if ((term_tmp - term_dict) > IT_MAX_WORD)
 759                 break;
 760         }
 761         *term_tmp++ = ')';
 762         *term_tmp = '\0';
 763         break;
 764     case 2:
 765         if (!term_100 (zh->reg->zebra_maps, reg_type,
 766                        term_sub, term_component,
 767                        space_split, term_dst))
 768             return 0;
 769         yaz_log(log_level_rpn, "Relation <=");
 770
 771         *term_tmp++ = '(';
 772         for (i = 0; term_component[i]; )
 773         {
 774             int j = 0;
 775
 776             while (j < i)
 777                 string_rel_add_char (&term_tmp, term_component, &j);
 778             *term_tmp++ = '[';
 779
 780             *term_tmp++ = '^';
 781             string_rel_add_char (&term_tmp, term_component, &i);
 782             *term_tmp++ = '-';
 783
 784             *term_tmp++ = ']';
 785             *term_tmp++ = '.';
 786             *term_tmp++ = '*';
 787
 788             *term_tmp++ = '|';
 789
 790             if ((term_tmp - term_dict) > IT_MAX_WORD)
 791                 break;
 792         }
 793         for (i = 0; term_component[i]; )
 794             string_rel_add_char (&term_tmp, term_component, &i);
 795         *term_tmp++ = ')';
 796         *term_tmp = '\0';
 797         break;
 798     case 5:
 799         if (!term_100 (zh->reg->zebra_maps, reg_type,
 800                        term_sub, term_component, space_split, term_dst))
 801             return 0;
 802         yaz_log(log_level_rpn, "Relation >");
 803
 804         *term_tmp++ = '(';
 805         for (i = 0; term_component[i];)
 806         {
 807             int j = 0;
 808
 809             while (j < i)
 810                 string_rel_add_char (&term_tmp, term_component, &j);
 811             *term_tmp++ = '[';
 812
 813             *term_tmp++ = '^';
 814             *term_tmp++ = '-';
 815             string_rel_add_char (&term_tmp, term_component, &i);
 816
 817             *term_tmp++ = ']';
 818             *term_tmp++ = '.';
 819             *term_tmp++ = '*';
 820
 821             *term_tmp++ = '|';
 822
 823             if ((term_tmp - term_dict) > IT_MAX_WORD)
 824                 break;
 825         }
 826         for (i = 0; term_component[i];)
 827             string_rel_add_char (&term_tmp, term_component, &i);
 828         *term_tmp++ = '.';
 829         *term_tmp++ = '+';
 830         *term_tmp++ = ')';
 831         *term_tmp = '\0';
 832         break;
 833     case 4:
 834         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 835                        term_component, space_split, term_dst))
 836             return 0;
 837         yaz_log(log_level_rpn, "Relation >=");
 838
 839         *term_tmp++ = '(';
 840         for (i = 0; term_component[i];)
 841         {
 842             int j = 0;
 843
 844             if (i)
 845                 *term_tmp++ = '|';
 846             while (j < i)
 847                 string_rel_add_char (&term_tmp, term_component, &j);
 848             *term_tmp++ = '[';
 849
 850             if (term_component[i+1])
 851             {
 852                 *term_tmp++ = '^';
 853                 *term_tmp++ = '-';
 854                 string_rel_add_char (&term_tmp, term_component, &i);
 855             }
 856             else
 857             {
 858                 string_rel_add_char (&term_tmp, term_component, &i);
 859                 *term_tmp++ = '-';
 860             }
 861             *term_tmp++ = ']';
 862             *term_tmp++ = '.';
 863             *term_tmp++ = '*';
 864
 865             if ((term_tmp - term_dict) > IT_MAX_WORD)
 866                 break;
 867         }
 868         *term_tmp++ = ')';
 869         *term_tmp = '\0';
 870         break;
 871     case 3:
 872     default:
 873         yaz_log(log_level_rpn, "Relation =");
 874         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 875                        term_component, space_split, term_dst))
 876             return 0;
 877         strcat(term_tmp, "(");
 878         strcat(term_tmp, term_component);
 879         strcat(term_tmp, ")");
 880     }
 881     return 1;
 882 }
 883
 884 static int string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 885                         const char **term_sub,
 886                         oid_value attributeSet, NMEM stream,
 887                         struct grep_info *grep_info,
 888                         int reg_type, int complete_flag,
 889                         int num_bases, char **basenames,
 890                         char *term_dst, int xpath_use);
 891
 892 static RSET term_trunc(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 893                         const char **term_sub,
 894                         oid_value attributeSet, NMEM stream,
 895                         struct grep_info *grep_info,
 896                         int reg_type, int complete_flag,
 897                         int num_bases, char **basenames,
 898                         char *term_dst,
 899                         const char *rank_type, int xpath_use,
 900                         NMEM rset_nmem)
 901 {
 902     int r;
 903     grep_info->isam_p_indx = 0;
 904     r = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 905                      reg_type, complete_flag, num_bases, basenames,
 906                      term_dst, xpath_use);
 907     if (r < 1)
 908         return 0;
 909     yaz_log(log_level_rpn, "term: %s", term_dst);
 910     return rset_trunc(zh, grep_info->isam_p_buf,
 911                        grep_info->isam_p_indx, term_dst,
 912                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 913                        zapt->term->which, rset_nmem,
 914                        key_it_ctrl,key_it_ctrl->scope);
 915 }
 916 static char *nmem_strdup_i(NMEM nmem, int v)
 917 {
 918     char val_str[64];
 919     sprintf (val_str, "%d", v);
 920     return nmem_strdup(nmem, val_str);
 921 }
 922
 923 static int string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 924                        const char **term_sub,
 925                        oid_value attributeSet, NMEM stream,
 926                        struct grep_info *grep_info,
 927                        int reg_type, int complete_flag,
 928                        int num_bases, char **basenames,
 929                        char *term_dst, int xpath_use)
 930 {
 931     char term_dict[2*IT_MAX_WORD+4000];
 932     int j, r, base_no;
 933     AttrType truncation;
 934     int truncation_value;
 935     AttrType use;
 936     int use_value;
 937     const char *use_string = 0;
 938     oid_value curAttributeSet = attributeSet;
 939     const char *termp;
 940     struct rpn_char_map_info rcmi;
 941     int space_split = complete_flag ? 0 : 1;
 942
 943     int bases_ok = 0;     /* no of databases with OK attribute */
 944     int errCode = 0;      /* err code (if any is not OK) */
 945     char *errString = 0;  /* addinfo */
 946
 947     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
 948     attr_init (&use, zapt, 1);
 949     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
 950     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
 951     attr_init (&truncation, zapt, 5);
 952     truncation_value = attr_find (&truncation, NULL);
 953     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
 954
 955     if (use_value == -1)    /* no attribute - assumy "any" */
 956         use_value = 1016;
 957     for (base_no = 0; base_no < num_bases; base_no++)
 958     {
 959         int attr_ok = 0;
 960         int regex_range = 0;
 961         int init_pos = 0;
 962         attent attp;
 963         data1_local_attribute id_xpath_attr;
 964         data1_local_attribute *local_attr;
 965         int max_pos, prefix_len = 0;
 966
 967         termp = *term_sub;
 968
 969         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
 970         {
 971             zh->errCode = 109; /* Database unavailable */
 972             zh->errString = basenames[base_no];
 973             return -1;
 974         }
 975         if (xpath_use > 0 && use_value == -2)
 976         {
 977             use_value = xpath_use;
 978             attp.local_attributes = &id_xpath_attr;
 979             attp.attset_ordinal = VAL_IDXPATH;
 980             id_xpath_attr.next = 0;
 981             id_xpath_attr.local = use_value;
 982         }
 983         else if (curAttributeSet == VAL_IDXPATH)
 984         {
 985             attp.local_attributes = &id_xpath_attr;
 986             attp.attset_ordinal = VAL_IDXPATH;
 987             id_xpath_attr.next = 0;
 988             id_xpath_attr.local = use_value;
 989         }
 990         else
 991         {
 992             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
 993                                             use_string)))
 994             {
 995                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
 996                       curAttributeSet, use_value, r);
 997                 if (r == -1)
 998                 {
 999                     /* set was found, but value wasn't defined */
1000                     errCode = 114;
1001                     if (use_string)
1002                         errString = nmem_strdup(stream, use_string);
1003                     else
1004                         errString = nmem_strdup_i (stream, use_value);
1005                 }
1006                 else
1007                 {
1008                     int oid[OID_SIZE];
1009                     struct oident oident;
1010
1011                     oident.proto = PROTO_Z3950;
1012                     oident.oclass = CLASS_ATTSET;
1013                     oident.value = curAttributeSet;
1014                     oid_ent_to_oid (&oident, oid);
1015
1016                     errCode = 121;
1017                     errString = nmem_strdup (stream, oident.desc);
1018                 }
1019                 continue;
1020             }
1021         }
1022         for (local_attr = attp.local_attributes; local_attr;
1023              local_attr = local_attr->next)
1024         {
1025             int ord;
1026             char ord_buf[32];
1027             int i, ord_len;
1028
1029             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
1030                                          local_attr->local);
1031             if (ord < 0)
1032                 continue;
1033             if (prefix_len)
1034                 term_dict[prefix_len++] = '|';
1035             else
1036                 term_dict[prefix_len++] = '(';
1037
1038             ord_len = key_SU_encode (ord, ord_buf);
1039             for (i = 0; i<ord_len; i++)
1040             {
1041                 term_dict[prefix_len++] = 1;
1042                 term_dict[prefix_len++] = ord_buf[i];
1043             }
1044         }
1045         if (!prefix_len)
1046         {
1047 #if 1
1048             bases_ok++;
1049 #else
1050             errCode = 114;
1051             errString = nmem_strdup_i(stream, use_value);
1052             continue;
1053 #endif
1054         }
1055         else
1056         {
1057             bases_ok++; /* this has OK attributes */
1058             attr_ok = 1;
1059         }
1060
1061         term_dict[prefix_len++] = ')';
1062         term_dict[prefix_len++] = 1;
1063         term_dict[prefix_len++] = reg_type;
1064         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1065         term_dict[prefix_len] = '\0';
1066         j = prefix_len;
1067         switch (truncation_value)
1068         {
1069         case -1:         /* not specified */
1070         case 100:        /* do not truncate */
1071             if (!string_relation (zh, zapt, &termp, term_dict,
1072                                   attributeSet,
1073                                   reg_type, space_split, term_dst))
1074                 return 0;
1075             break;
1076         case 1:          /* right truncation */
1077             term_dict[j++] = '(';
1078             if (!term_100(zh->reg->zebra_maps, reg_type,
1079                           &termp, term_dict + j, space_split, term_dst))
1080                 return 0;
1081             strcat(term_dict, ".*)");
1082             break;
1083         case 2:          /* keft truncation */
1084             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1085             if (!term_100(zh->reg->zebra_maps, reg_type,
1086                           &termp, term_dict + j, space_split, term_dst))
1087                 return 0;
1088             strcat(term_dict, ")");
1089             break;
1090         case 3:          /* left&right truncation */
1091             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1092             if (!term_100(zh->reg->zebra_maps, reg_type,
1093                           &termp, term_dict + j, space_split, term_dst))
1094                 return 0;
1095             strcat(term_dict, ".*)");
1096             break;
1097         case 101:        /* process # in term */
1098             term_dict[j++] = '(';
1099             if (!term_101(zh->reg->zebra_maps, reg_type,
1100                           &termp, term_dict + j, space_split, term_dst))
1101                 return 0;
1102             strcat(term_dict, ")");
1103             break;
1104         case 102:        /* Regexp-1 */
1105             term_dict[j++] = '(';
1106             if (!term_102(zh->reg->zebra_maps, reg_type,
1107                           &termp, term_dict + j, space_split, term_dst))
1108                 return 0;
1109             strcat(term_dict, ")");
1110             break;
1111         case 103:       /* Regexp-2 */
1112             r = 1;
1113             term_dict[j++] = '(';
1114             init_pos = 2;
1115             if (!term_103 (zh->reg->zebra_maps, reg_type,
1116                            &termp, term_dict + j, &regex_range,
1117                            space_split, term_dst))
1118                 return 0;
1119             strcat(term_dict, ")");
1120         case 104:        /* process # and ! in term */
1121             term_dict[j++] = '(';
1122             if (!term_104 (zh->reg->zebra_maps, reg_type,
1123                            &termp, term_dict + j, space_split, term_dst))
1124                 return 0;
1125             strcat(term_dict, ")");
1126             break;
1127         case 105:        /* process * and ! in term */
1128             term_dict[j++] = '(';
1129             if (!term_105 (zh->reg->zebra_maps, reg_type,
1130                            &termp, term_dict + j, space_split, term_dst, 1))
1131                 return 0;
1132             strcat(term_dict, ")");
1133             break;
1134         case 106:        /* process * and ! in term */
1135             term_dict[j++] = '(';
1136             if (!term_105 (zh->reg->zebra_maps, reg_type,
1137                            &termp, term_dict + j, space_split, term_dst, 0))
1138                 return 0;
1139             strcat(term_dict, ")");
1140             break;
1141         default:
1142             zh->errCode = 120;
1143             zh->errString = nmem_strdup_i(stream, truncation_value);
1144             return -1;
1145         }
1146         if (attr_ok)
1147         {
1148             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1149             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1150                                  grep_info, &max_pos, init_pos,
1151                                  grep_handle);
1152             if (r)
1153                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1154         }
1155     }
1156     if (!bases_ok)
1157     {
1158         zh->errCode = errCode;
1159         zh->errString = errString;
1160         return -1;
1161     }
1162     *term_sub = termp;
1163     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1164     return 1;
1165 }
1166
1167
1168 /* convert APT search term to UTF8 */
1169 static int zapt_term_to_utf8 (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1170                               char *termz)
1171 {
1172     size_t sizez;
1173     Z_Term *term = zapt->term;
1174
1175     switch (term->which)
1176     {
1177     case Z_Term_general:
1178         if (zh->iconv_to_utf8 != 0)
1179         {
1180             char *inbuf = term->u.general->buf;
1181             size_t inleft = term->u.general->len;
1182             char *outbuf = termz;
1183             size_t outleft = IT_MAX_WORD-1;
1184             size_t ret;
1185
1186             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1187                         &outbuf, &outleft);
1188             if (ret == (size_t)(-1))
1189             {
1190                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1191                 zh->errCode = 125;
1192                 return -1;
1193             }
1194             *outbuf = 0;
1195         }
1196         else
1197         {
1198             sizez = term->u.general->len;
1199             if (sizez > IT_MAX_WORD-1)
1200                 sizez = IT_MAX_WORD-1;
1201             memcpy (termz, term->u.general->buf, sizez);
1202             termz[sizez] = '\0';
1203         }
1204         break;
1205     case Z_Term_characterString:
1206         sizez = strlen(term->u.characterString);
1207         if (sizez > IT_MAX_WORD-1)
1208             sizez = IT_MAX_WORD-1;
1209         memcpy (termz, term->u.characterString, sizez);
1210         termz[sizez] = '\0';
1211         break;
1212     default:
1213         zh->errCode = 124;
1214         return -1;
1215     }
1216     return 0;
1217 }
1218
1219 /* convert APT SCAN term to internal cmap */
1220 static int trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1221                             char *termz, int reg_type)
1222 {
1223     char termz0[IT_MAX_WORD];
1224
1225     if (zapt_term_to_utf8(zh, zapt, termz0))
1226         return -1;    /* error */
1227     else
1228     {
1229         const char **map;
1230         const char *cp = (const char *) termz0;
1231         const char *cp_end = cp + strlen(cp);
1232         const char *src;
1233         int i = 0;
1234         const char *space_map = NULL;
1235         int len;
1236
1237         while ((len = (cp_end - cp)) > 0)
1238         {
1239             map = zebra_maps_input (zh->reg->zebra_maps, reg_type, &cp, len, 0);
1240             if (**map == *CHR_SPACE)
1241                 space_map = *map;
1242             else
1243             {
1244                 if (i && space_map)
1245                     for (src = space_map; *src; src++)
1246                         termz[i++] = *src;
1247                 space_map = NULL;
1248                 for (src = *map; *src; src++)
1249                     termz[i++] = *src;
1250             }
1251         }
1252         termz[i] = '\0';
1253     }
1254     return 0;
1255 }
1256
1257 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1258                      const char *termz, NMEM stream, unsigned reg_id)
1259 {
1260     WRBUF wrbuf = 0;
1261     AttrType truncation;
1262     int truncation_value;
1263     char *ex_list = 0;
1264
1265     attr_init (&truncation, zapt, 5);
1266     truncation_value = attr_find (&truncation, NULL);
1267
1268     switch (truncation_value)
1269     {
1270     default:
1271         ex_list = "";
1272         break;
1273     case 101:
1274         ex_list = "#";
1275         break;
1276     case 102:
1277     case 103:
1278         ex_list = 0;
1279         break;
1280     case 104:
1281         ex_list = "!#";
1282         break;
1283     case 105:
1284         ex_list = "!*";
1285         break;
1286     }
1287     if (ex_list)
1288         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1289                               termz, strlen(termz));
1290     if (!wrbuf)
1291         return nmem_strdup(stream, termz);
1292     else
1293     {
1294         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1295         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1296         buf[wrbuf_len(wrbuf)] = '\0';
1297         return buf;
1298     }
1299 }
1300
1301 static void grep_info_delete (struct grep_info *grep_info)
1302 {
1303 #ifdef TERM_COUNT
1304     xfree(grep_info->term_no);
1305 #endif
1306     xfree (grep_info->isam_p_buf);
1307 }
1308
1309 static int grep_info_prepare (ZebraHandle zh,
1310                               Z_AttributesPlusTerm *zapt,
1311                               struct grep_info *grep_info,
1312                               int reg_type,
1313                               NMEM stream)
1314 {
1315     AttrType termset;
1316     int termset_value_numeric;
1317     const char *termset_value_string;
1318
1319 #ifdef TERM_COUNT
1320     grep_info->term_no = 0;
1321 #endif
1322     grep_info->isam_p_size = 0;
1323     grep_info->isam_p_buf = NULL;
1324     grep_info->zh = zh;
1325     grep_info->reg_type = reg_type;
1326     grep_info->termset = 0;
1327
1328     if (!zapt)
1329         return 0;
1330     attr_init (&termset, zapt, 8);
1331     termset_value_numeric =
1332         attr_find_ex (&termset, NULL, &termset_value_string);
1333     if (termset_value_numeric != -1)
1334     {
1335         char resname[32];
1336         const char *termset_name = 0;
1337         if (termset_value_numeric != -2)
1338         {
1339
1340             sprintf (resname, "%d", termset_value_numeric);
1341             termset_name = resname;
1342         }
1343         else
1344             termset_name = termset_value_string;
1345         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1346         grep_info->termset = resultSetAdd (zh, termset_name, 1);
1347         if (!grep_info->termset)
1348         {
1349             zh->errCode = 128;
1350             zh->errString = nmem_strdup (stream, termset_name);
1351             return -1;
1352         }
1353     }
1354     return 0;
1355 }
1356
1357
1358 static RSET rpn_search_APT_phrase (ZebraHandle zh,
1359                                    Z_AttributesPlusTerm *zapt,
1360                                    const char *termz_org,
1361                                    oid_value attributeSet,
1362                                    NMEM stream,
1363                                    int reg_type, int complete_flag,
1364                                    const char *rank_type, int xpath_use,
1365                                    int num_bases, char **basenames,
1366                                    NMEM rset_nmem)
1367 {
1368     char term_dst[IT_MAX_WORD+1];
1369     RSET rset[TERM_LIST_LENGTH_MAX], result;
1370     size_t rset_no = 0;
1371     struct grep_info grep_info;
1372     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1373     const char *termp = termz;
1374
1375     *term_dst = 0;
1376     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1377         return 0;
1378     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1379     {
1380         yaz_log(log_level_rpn, "APT_phrase termp=%s", termp);
1381         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1382                                     stream, &grep_info,
1383                                     reg_type, complete_flag,
1384                                     num_bases, basenames,
1385                                     term_dst, rank_type,
1386                                     xpath_use,rset_nmem);
1387         if (!rset[rset_no])
1388             break;
1389     }
1390     grep_info_delete (&grep_info);
1391     if (rset_no == 0)
1392         return rsnull_create (rset_nmem,key_it_ctrl);
1393     else if (rset_no == 1)
1394         return (rset[0]);
1395     else
1396         result = rsprox_create( rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1397                        rset_no, rset,
1398                        1 /* ordered */, 0 /* exclusion */,
1399                        3 /* relation */, 1 /* distance */);
1400     return result;
1401 }
1402
1403 static RSET rpn_search_APT_or_list (ZebraHandle zh,
1404                                     Z_AttributesPlusTerm *zapt,
1405                                     const char *termz_org,
1406                                     oid_value attributeSet,
1407                                     NMEM stream,
1408                                     int reg_type, int complete_flag,
1409                                     const char *rank_type,
1410                                     int xpath_use,
1411                                     int num_bases, char **basenames,
1412                                     NMEM rset_nmem)
1413 {
1414     char term_dst[IT_MAX_WORD+1];
1415     RSET rset[TERM_LIST_LENGTH_MAX];
1416     size_t rset_no = 0;
1417     struct grep_info grep_info;
1418     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1419     const char *termp = termz;
1420
1421     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1422         return 0;
1423     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1424     {
1425         yaz_log(log_level_rpn, "APT_or_list termp=%s", termp);
1426         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1427                                     stream, &grep_info,
1428                                     reg_type, complete_flag,
1429                                     num_bases, basenames,
1430                                     term_dst, rank_type,
1431                                     xpath_use,rset_nmem);
1432         if (!rset[rset_no])
1433             break;
1434     }
1435     grep_info_delete (&grep_info);
1436     if (rset_no == 0)
1437         return rsnull_create (rset_nmem,key_it_ctrl);
1438     return rsmulti_or_create(rset_nmem, key_it_ctrl,key_it_ctrl->scope,
1439                              rset_no, rset);
1440 }
1441
1442 static RSET rpn_search_APT_and_list (ZebraHandle zh,
1443                                      Z_AttributesPlusTerm *zapt,
1444                                      const char *termz_org,
1445                                      oid_value attributeSet,
1446                                      NMEM stream,
1447                                      int reg_type, int complete_flag,
1448                                      const char *rank_type,
1449                                      int xpath_use,
1450                                      int num_bases, char **basenames,
1451                                      NMEM rset_nmem)
1452 {
1453     char term_dst[IT_MAX_WORD+1];
1454     RSET rset[TERM_LIST_LENGTH_MAX];
1455     size_t rset_no = 0;
1456     struct grep_info grep_info;
1457     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1458     const char *termp = termz;
1459
1460     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1461         return 0;
1462     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1463     {
1464         yaz_log(log_level_rpn, "APT_and_list termp=%s", termp);
1465         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1466                                     stream, &grep_info,
1467                                     reg_type, complete_flag,
1468                                     num_bases, basenames,
1469                                     term_dst, rank_type,
1470                                     xpath_use, rset_nmem);
1471         if (!rset[rset_no])
1472             break;
1473     }
1474     grep_info_delete (&grep_info);
1475     if (rset_no == 0)
1476         return rsnull_create(rset_nmem,key_it_ctrl);
1477
1478     return rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1479                               rset_no, rset);
1480 }
1481
1482 static int numeric_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1483                              const char **term_sub,
1484                              char *term_dict,
1485                              oid_value attributeSet,
1486                              struct grep_info *grep_info,
1487                              int *max_pos,
1488                              int reg_type,
1489                              char *term_dst)
1490 {
1491     AttrType relation;
1492     int relation_value;
1493     int term_value;
1494     int r;
1495     char *term_tmp = term_dict + strlen(term_dict);
1496
1497     attr_init (&relation, zapt, 2);
1498     relation_value = attr_find (&relation, NULL);
1499
1500     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1501
1502     if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1503                    term_dst))
1504         return 0;
1505     term_value = atoi (term_tmp);
1506     switch (relation_value)
1507     {
1508     case 1:
1509         yaz_log(log_level_rpn, "Relation <");
1510         gen_regular_rel (term_tmp, term_value-1, 1);
1511         break;
1512     case 2:
1513         yaz_log(log_level_rpn, "Relation <=");
1514         gen_regular_rel (term_tmp, term_value, 1);
1515         break;
1516     case 4:
1517         yaz_log(log_level_rpn, "Relation >=");
1518         gen_regular_rel (term_tmp, term_value, 0);
1519         break;
1520     case 5:
1521         yaz_log(log_level_rpn, "Relation >");
1522         gen_regular_rel (term_tmp, term_value+1, 0);
1523         break;
1524     case 3:
1525     default:
1526         yaz_log(log_level_rpn, "Relation =");
1527         sprintf (term_tmp, "(0*%d)", term_value);
1528     }
1529     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1530     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1531                           0, grep_handle);
1532     if (r)
1533         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1534     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1535     return 1;
1536 }
1537
1538 static int numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1539                          const char **term_sub,
1540                          oid_value attributeSet, struct grep_info *grep_info,
1541                          int reg_type, int complete_flag,
1542                          int num_bases, char **basenames,
1543                          char *term_dst, int xpath_use, NMEM stream)
1544 {
1545     char term_dict[2*IT_MAX_WORD+2];
1546     int r, base_no;
1547     AttrType use;
1548     int use_value;
1549     const char *use_string = 0;
1550     oid_value curAttributeSet = attributeSet;
1551     const char *termp;
1552     struct rpn_char_map_info rcmi;
1553
1554     int bases_ok = 0;     /* no of databases with OK attribute */
1555     int errCode = 0;      /* err code (if any is not OK) */
1556     char *errString = 0;  /* addinfo */
1557
1558     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1559     attr_init (&use, zapt, 1);
1560     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1561
1562     if (use_value == -1)
1563         use_value = 1016;
1564
1565     for (base_no = 0; base_no < num_bases; base_no++)
1566     {
1567         attent attp;
1568         data1_local_attribute id_xpath_attr;
1569         data1_local_attribute *local_attr;
1570         int max_pos, prefix_len = 0;
1571
1572         termp = *term_sub;
1573         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1574         {
1575             use_value = xpath_use;
1576             attp.local_attributes = &id_xpath_attr;
1577             attp.attset_ordinal = VAL_IDXPATH;
1578             id_xpath_attr.next = 0;
1579             id_xpath_attr.local = use_value;
1580         }
1581         else if (curAttributeSet == VAL_IDXPATH)
1582         {
1583             attp.local_attributes = &id_xpath_attr;
1584             attp.attset_ordinal = VAL_IDXPATH;
1585             id_xpath_attr.next = 0;
1586             id_xpath_attr.local = use_value;
1587         }
1588         else
1589         {
1590             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1591                                             use_string)))
1592             {
1593                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1594                       curAttributeSet, use_value, r);
1595                 if (r == -1)
1596                 {
1597                     errString = nmem_strdup_i(stream, use_value);
1598                     errCode = 114;
1599                 }
1600                 else
1601                     errCode = 121;
1602                 continue;
1603             }
1604         }
1605         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1606         {
1607             zh->errCode = 109; /* Database unavailable */
1608             zh->errString = basenames[base_no];
1609             return -1;
1610         }
1611         for (local_attr = attp.local_attributes; local_attr;
1612              local_attr = local_attr->next)
1613         {
1614             int ord;
1615             char ord_buf[32];
1616             int i, ord_len;
1617
1618             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
1619                                           local_attr->local);
1620             if (ord < 0)
1621                 continue;
1622             if (prefix_len)
1623                 term_dict[prefix_len++] = '|';
1624             else
1625                 term_dict[prefix_len++] = '(';
1626
1627             ord_len = key_SU_encode (ord, ord_buf);
1628             for (i = 0; i<ord_len; i++)
1629             {
1630                 term_dict[prefix_len++] = 1;
1631                 term_dict[prefix_len++] = ord_buf[i];
1632             }
1633         }
1634         if (!prefix_len)
1635         {
1636             errCode = 114;
1637             errString = nmem_strdup_i(stream, use_value);
1638             continue;
1639         }
1640         bases_ok++;
1641         term_dict[prefix_len++] = ')';
1642         term_dict[prefix_len++] = 1;
1643         term_dict[prefix_len++] = reg_type;
1644         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1645         term_dict[prefix_len] = '\0';
1646         if (!numeric_relation (zh, zapt, &termp, term_dict,
1647                                attributeSet, grep_info, &max_pos, reg_type,
1648                                term_dst))
1649             return 0;
1650     }
1651     if (!bases_ok)
1652     {
1653         zh->errCode = errCode;
1654         zh->errString = errString;
1655         return -1;
1656     }
1657     *term_sub = termp;
1658     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1659     return 1;
1660 }
1661
1662 static RSET rpn_search_APT_numeric (ZebraHandle zh,
1663                                     Z_AttributesPlusTerm *zapt,
1664                                     const char *termz,
1665                                     oid_value attributeSet,
1666                                     NMEM stream,
1667                                     int reg_type, int complete_flag,
1668                                     const char *rank_type, int xpath_use,
1669                                     int num_bases, char **basenames,
1670                                     NMEM rset_nmem)
1671 {
1672     char term_dst[IT_MAX_WORD+1];
1673     const char *termp = termz;
1674     RSET rset[TERM_LIST_LENGTH_MAX];
1675     int  r;
1676     size_t rset_no = 0;
1677     struct grep_info grep_info;
1678
1679     yaz_log(log_level_rpn, "APT_numeric t='%s'",termz);
1680     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1681         return 0;
1682     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1683     {
1684         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1685         grep_info.isam_p_indx = 0;
1686         r = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1687                           reg_type, complete_flag, num_bases, basenames,
1688                           term_dst, xpath_use,
1689                           stream);
1690         if (r < 1)
1691             break;
1692         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1693         rset[rset_no] = rset_trunc(zh, grep_info.isam_p_buf,
1694                                     grep_info.isam_p_indx, term_dst,
1695                                     strlen(term_dst), rank_type,
1696                                     0 /* preserve position */,
1697                                     zapt->term->which, rset_nmem,
1698                                     key_it_ctrl,key_it_ctrl->scope);
1699         if (!rset[rset_no])
1700             break;
1701     }
1702     grep_info_delete (&grep_info);
1703     if (rset_no == 0)
1704         return rsnull_create(rset_nmem,key_it_ctrl);
1705     if (rset_no == 1)
1706         return rset[0];
1707     return rsmulti_and_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1708                               rset_no, rset);
1709 }
1710
1711 static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1712                                   const char *termz,
1713                                   oid_value attributeSet,
1714                                   NMEM stream,
1715                                   const char *rank_type, NMEM rset_nmem)
1716 {
1717     RSET result;
1718     RSFD rsfd;
1719     struct it_key key;
1720     int sys;
1721     result = rstemp_create( rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1722                      res_get (zh->res, "setTmpDir"),0 );
1723     rsfd = rset_open (result, RSETF_WRITE);
1724
1725     sys = atoi(termz);
1726     if (sys <= 0)
1727         sys = 1;
1728     key.mem[0] = sys;
1729     key.mem[1] = 1;
1730     key.len = 2;
1731     rset_write (rsfd, &key);
1732     rset_close (rsfd);
1733     return result;
1734 }
1735
1736 static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1737                            oid_value attributeSet, NMEM stream,
1738                            Z_SortKeySpecList *sort_sequence,
1739                            const char *rank_type)
1740 {
1741     int i;
1742     int sort_relation_value;
1743     AttrType sort_relation_type;
1744     int use_value;
1745     AttrType use_type;
1746     Z_SortKeySpec *sks;
1747     Z_SortKey *sk;
1748     Z_AttributeElement *ae;
1749     int oid[OID_SIZE];
1750     oident oe;
1751     char termz[20];
1752
1753     attr_init (&sort_relation_type, zapt, 7);
1754     sort_relation_value = attr_find (&sort_relation_type, &attributeSet);
1755
1756     attr_init (&use_type, zapt, 1);
1757     use_value = attr_find (&use_type, &attributeSet);
1758
1759     if (!sort_sequence->specs)
1760     {
1761         sort_sequence->num_specs = 10;
1762         sort_sequence->specs = (Z_SortKeySpec **)
1763             nmem_malloc(stream, sort_sequence->num_specs *
1764                          sizeof(*sort_sequence->specs));
1765         for (i = 0; i<sort_sequence->num_specs; i++)
1766             sort_sequence->specs[i] = 0;
1767     }
1768     if (zapt->term->which != Z_Term_general)
1769         i = 0;
1770     else
1771         i = atoi_n ((char *) zapt->term->u.general->buf,
1772                     zapt->term->u.general->len);
1773     if (i >= sort_sequence->num_specs)
1774         i = 0;
1775     sprintf (termz, "%d", i);
1776
1777     oe.proto = PROTO_Z3950;
1778     oe.oclass = CLASS_ATTSET;
1779     oe.value = attributeSet;
1780     if (!oid_ent_to_oid (&oe, oid))
1781         return 0;
1782
1783     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1784     sks->sortElement = (Z_SortElement *)
1785         nmem_malloc(stream, sizeof(*sks->sortElement));
1786     sks->sortElement->which = Z_SortElement_generic;
1787     sk = sks->sortElement->u.generic = (Z_SortKey *)
1788         nmem_malloc(stream, sizeof(*sk));
1789     sk->which = Z_SortKey_sortAttributes;
1790     sk->u.sortAttributes = (Z_SortAttributes *)
1791         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1792
1793     sk->u.sortAttributes->id = oid;
1794     sk->u.sortAttributes->list = (Z_AttributeList *)
1795         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
1796     sk->u.sortAttributes->list->num_attributes = 1;
1797     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
1798         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
1799     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
1800         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
1801     ae->attributeSet = 0;
1802     ae->attributeType = (int *)
1803         nmem_malloc(stream, sizeof(*ae->attributeType));
1804     *ae->attributeType = 1;
1805     ae->which = Z_AttributeValue_numeric;
1806     ae->value.numeric = (int *)
1807         nmem_malloc(stream, sizeof(*ae->value.numeric));
1808     *ae->value.numeric = use_value;
1809
1810     sks->sortRelation = (int *)
1811         nmem_malloc(stream, sizeof(*sks->sortRelation));
1812     if (sort_relation_value == 1)
1813         *sks->sortRelation = Z_SortKeySpec_ascending;
1814     else if (sort_relation_value == 2)
1815         *sks->sortRelation = Z_SortKeySpec_descending;
1816     else
1817         *sks->sortRelation = Z_SortKeySpec_ascending;
1818
1819     sks->caseSensitivity = (int *)
1820         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1821     *sks->caseSensitivity = 0;
1822
1823     sks->which = Z_SortKeySpec_null;
1824     sks->u.null = odr_nullval ();
1825     sort_sequence->specs[i] = sks;
1826     return rsnull_create (NULL,key_it_ctrl);
1827         /* FIXME - nmem?? */
1828 }
1829
1830
1831 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1832                        oid_value attributeSet,
1833                        struct xpath_location_step *xpath, int max, NMEM mem)
1834 {
1835     oid_value curAttributeSet = attributeSet;
1836     AttrType use;
1837     const char *use_string = 0;
1838
1839     attr_init (&use, zapt, 1);
1840     attr_find_ex (&use, &curAttributeSet, &use_string);
1841
1842     if (!use_string || *use_string != '/')
1843         return -1;
1844
1845     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1846 }
1847
1848
1849
1850 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1851                         int reg_type, const char *term, int use,
1852                         oid_value curAttributeSet, NMEM rset_nmem)
1853 {
1854     RSET rset;
1855     struct grep_info grep_info;
1856     char term_dict[2048];
1857     char ord_buf[32];
1858     int prefix_len = 0;
1859     int ord = zebraExplain_lookupSU (zh->reg->zei, curAttributeSet, use);
1860     int ord_len, i, r, max_pos;
1861     int term_type = Z_Term_characterString;
1862     const char *flags = "void";
1863
1864     if (grep_info_prepare (zh, 0 /* zapt */, &grep_info, '0', stream))
1865         return rsnull_create (rset_nmem,key_it_ctrl);
1866
1867     if (ord < 0)
1868         return rsnull_create (rset_nmem,key_it_ctrl);
1869     if (prefix_len)
1870         term_dict[prefix_len++] = '|';
1871     else
1872         term_dict[prefix_len++] = '(';
1873
1874     ord_len = key_SU_encode (ord, ord_buf);
1875     for (i = 0; i<ord_len; i++)
1876     {
1877         term_dict[prefix_len++] = 1;
1878         term_dict[prefix_len++] = ord_buf[i];
1879     }
1880     term_dict[prefix_len++] = ')';
1881     term_dict[prefix_len++] = 1;
1882     term_dict[prefix_len++] = reg_type;
1883
1884     strcpy(term_dict+prefix_len, term);
1885
1886     grep_info.isam_p_indx = 0;
1887     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
1888                           &grep_info, &max_pos, 0, grep_handle);
1889     yaz_log (YLOG_LOG, "%s %d positions", term,
1890              grep_info.isam_p_indx);
1891     rset = rset_trunc(zh, grep_info.isam_p_buf,
1892                        grep_info.isam_p_indx, term, strlen(term),
1893                        flags, 1, term_type,rset_nmem,
1894                        key_it_ctrl, key_it_ctrl->scope);
1895     grep_info_delete (&grep_info);
1896     return rset;
1897 }
1898
1899 static RSET rpn_search_xpath (ZebraHandle zh,
1900                               oid_value attributeSet,
1901                               int num_bases, char **basenames,
1902                               NMEM stream, const char *rank_type, RSET rset,
1903                               int xpath_len, struct xpath_location_step *xpath,
1904                               NMEM rset_nmem)
1905 {
1906     oid_value curAttributeSet = attributeSet;
1907     int base_no;
1908     int i;
1909
1910     if (xpath_len < 0)
1911         return rset;
1912
1913     yaz_log (YLOG_DEBUG, "xpath len=%d", xpath_len);
1914     for (i = 0; i<xpath_len; i++)
1915     {
1916         yaz_log (log_level_rpn, "XPATH %d %s", i, xpath[i].part);
1917
1918     }
1919
1920     curAttributeSet = VAL_IDXPATH;
1921
1922     /*
1923       //a    ->    a/.*
1924       //a/b  ->    b/a/.*
1925       /a     ->    a/
1926       /a/b   ->    b/a/
1927
1928       /      ->    none
1929
1930    a[@attr = value]/b[@other = othervalue]
1931
1932  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
1933  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
1934  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
1935  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
1936  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
1937  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
1938
1939     */
1940
1941     dict_grep_cmap (zh->reg->dict, 0, 0);
1942
1943     for (base_no = 0; base_no < num_bases; base_no++)
1944     {
1945         int level = xpath_len;
1946         int first_path = 1;
1947
1948         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1949         {
1950             zh->errCode = 109; /* Database unavailable */
1951             zh->errString = basenames[base_no];
1952             return rset;
1953         }
1954         while (--level >= 0)
1955         {
1956             char xpath_rev[128];
1957             int i, len;
1958             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
1959
1960             *xpath_rev = 0;
1961             len = 0;
1962             for (i = level; i >= 1; --i)
1963             {
1964                 const char *cp = xpath[i].part;
1965                 if (*cp)
1966                 {
1967                     for (;*cp; cp++)
1968                         if (*cp == '*')
1969                         {
1970                             memcpy (xpath_rev + len, "[^/]*", 5);
1971                             len += 5;
1972                         }
1973                         else if (*cp == ' ')
1974                         {
1975
1976                             xpath_rev[len++] = 1;
1977                             xpath_rev[len++] = ' ';
1978                         }
1979
1980                         else
1981                             xpath_rev[len++] = *cp;
1982                     xpath_rev[len++] = '/';
1983                 }
1984                 else if (i == 1)  /* // case */
1985                 {
1986                     xpath_rev[len++] = '.';
1987                     xpath_rev[len++] = '*';
1988                 }
1989             }
1990             xpath_rev[len] = 0;
1991
1992             if (xpath[level].predicate &&
1993                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
1994                 xpath[level].predicate->u.relation.name[0])
1995             {
1996                 WRBUF wbuf = wrbuf_alloc();
1997                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
1998                 if (xpath[level].predicate->u.relation.value)
1999                 {
2000                     const char *cp = xpath[level].predicate->u.relation.value;
2001                     wrbuf_putc(wbuf, '=');
2002
2003                     while (*cp)
2004                     {
2005                         if (strchr(REGEX_CHARS, *cp))
2006                             wrbuf_putc(wbuf, '\\');
2007                         wrbuf_putc(wbuf, *cp);
2008                         cp++;
2009                     }
2010                 }
2011                 wrbuf_puts(wbuf, "");
2012                 rset_attr = xpath_trunc(
2013                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2014                     curAttributeSet,rset_nmem);
2015                 wrbuf_free(wbuf, 1);
2016             }
2017             else
2018             {
2019                 if (!first_path)
2020                     continue;
2021             }
2022             yaz_log (log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2023             if (strlen(xpath_rev))
2024             {
2025                 rset_start_tag = xpath_trunc(zh, stream, '0',
2026                         xpath_rev, 1, curAttributeSet, rset_nmem);
2027
2028                 rset_end_tag = xpath_trunc(zh, stream, '0',
2029                         xpath_rev, 2, curAttributeSet, rset_nmem);
2030
2031                 rset = rsbetween_create(rset_nmem, key_it_ctrl,
2032                                         key_it_ctrl->scope,
2033                                         rset_start_tag, rset,
2034                                         rset_end_tag, rset_attr);
2035             }
2036             first_path = 0;
2037         }
2038     }
2039
2040     return rset;
2041 }
2042
2043
2044
2045 static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2046                             oid_value attributeSet, NMEM stream,
2047                             Z_SortKeySpecList *sort_sequence,
2048                             int num_bases, char **basenames,
2049                             NMEM rset_nmem)
2050 {
2051     unsigned reg_id;
2052     char *search_type = NULL;
2053     char rank_type[128];
2054     int complete_flag;
2055     int sort_flag;
2056     char termz[IT_MAX_WORD+1];
2057     RSET rset = 0;
2058     int xpath_len;
2059     int xpath_use = 0;
2060     struct xpath_location_step xpath[10];
2061
2062     if (!log_level_set)
2063     {
2064         log_level_rpn = yaz_log_module_level("rpn");
2065         log_level_set = 1;
2066     }
2067     zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2068                      rank_type, &complete_flag, &sort_flag);
2069
2070     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2071     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2072     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2073     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2074
2075     if (zapt_term_to_utf8(zh, zapt, termz))
2076         return 0;
2077
2078     if (sort_flag)
2079         return rpn_sort_spec (zh, zapt, attributeSet, stream, sort_sequence,
2080                               rank_type);
2081     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2082     if (xpath_len >= 0)
2083     {
2084         xpath_use = 1016;
2085         if (xpath[xpath_len-1].part[0] == '@')
2086             xpath_use = 1015;
2087     }
2088
2089     if (!strcmp (search_type, "phrase"))
2090     {
2091         rset = rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
2092                                       reg_id, complete_flag, rank_type,
2093                                       xpath_use,
2094                                       num_bases, basenames, rset_nmem);
2095     }
2096     else if (!strcmp (search_type, "and-list"))
2097     {
2098         rset = rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
2099                                         reg_id, complete_flag, rank_type,
2100                                         xpath_use,
2101                                         num_bases, basenames, rset_nmem);
2102     }
2103     else if (!strcmp (search_type, "or-list"))
2104     {
2105         rset = rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
2106                                        reg_id, complete_flag, rank_type,
2107                                        xpath_use,
2108                                        num_bases, basenames, rset_nmem);
2109     }
2110     else if (!strcmp (search_type, "local"))
2111     {
2112         rset = rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
2113                                      rank_type, rset_nmem);
2114     }
2115     else if (!strcmp (search_type, "numeric"))
2116     {
2117         rset = rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
2118                                        reg_id, complete_flag, rank_type,
2119                                        xpath_use,
2120                                        num_bases, basenames, rset_nmem);
2121     }
2122     else if (!strcmp (search_type, "always"))
2123     {
2124         rset = 0;
2125     }
2126     else
2127         zh->errCode = 118;
2128     return rpn_search_xpath (zh, attributeSet, num_bases, basenames,
2129                              stream, rank_type, rset,
2130                              xpath_len, xpath, rset_nmem);
2131 }
2132
2133 static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
2134                                   oid_value attributeSet,
2135                                   NMEM stream, NMEM rset_nmem,
2136                                   Z_SortKeySpecList *sort_sequence,
2137                                   int num_bases, char **basenames)
2138 {
2139     RSET r = NULL;
2140     if (zs->which == Z_RPNStructure_complex)
2141     {
2142         Z_Operator *zop = zs->u.complex->roperator;
2143         RSET rsets[2]; /* l and r argument */
2144
2145         rsets[0]=rpn_search_structure (zh, zs->u.complex->s1,
2146                                        attributeSet, stream, rset_nmem,
2147                                        sort_sequence,
2148                                        num_bases, basenames);
2149         if (rsets[0] == NULL)
2150             return NULL;
2151         rsets[1]=rpn_search_structure (zh, zs->u.complex->s2,
2152                                        attributeSet, stream, rset_nmem,
2153                                        sort_sequence,
2154                                        num_bases, basenames);
2155         if (rsets[1] == NULL)
2156         {
2157             rset_delete (rsets[0]);
2158             return NULL;
2159         }
2160
2161         switch (zop->which)
2162         {
2163         case Z_Operator_and:
2164             r = rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2165                                    2, rsets);
2166             break;
2167         case Z_Operator_or:
2168             r = rsmulti_or_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2169                                   2, rsets);
2170             break;
2171         case Z_Operator_and_not:
2172             r = rsbool_create_not(rset_nmem,key_it_ctrl, key_it_ctrl->scope,
2173                     rsets[0],rsets[1]);
2174             break;
2175         case Z_Operator_prox:
2176             if (zop->u.prox->which != Z_ProximityOperator_known)
2177             {
2178                 zh->errCode = 132;
2179                 return NULL;
2180             }
2181             if (*zop->u.prox->u.known != Z_ProxUnit_word)
2182             {
2183                 char *val = (char *) nmem_malloc(stream, 16);
2184                 zh->errCode = 132;
2185                 zh->errString = val;
2186                 sprintf (val, "%d", *zop->u.prox->u.known);
2187                 return NULL;
2188             }
2189             else
2190             {
2191                 /* new / old prox */
2192                 r = rsprox_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
2193                          2, rsets,
2194                          *zop->u.prox->ordered,
2195                          (!zop->u.prox->exclusion ?
2196                               0 : *zop->u.prox->exclusion),
2197                          *zop->u.prox->relationType,
2198                          *zop->u.prox->distance );
2199             }
2200             break;
2201         default:
2202             zh->errCode = 110;
2203             return NULL;
2204         }
2205     }
2206     else if (zs->which == Z_RPNStructure_simple)
2207     {
2208         if (zs->u.simple->which == Z_Operand_APT)
2209         {
2210             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2211             r = rpn_search_APT (zh, zs->u.simple->u.attributesPlusTerm,
2212                                 attributeSet, stream, sort_sequence,
2213                                 num_bases, basenames,rset_nmem);
2214         }
2215         else if (zs->u.simple->which == Z_Operand_resultSetId)
2216         {
2217             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2218             r = resultSetRef (zh, zs->u.simple->u.resultSetId);
2219             if (!r)
2220             {
2221                 r = rsnull_create (rset_nmem,key_it_ctrl);
2222                 zh->errCode = 30;
2223                 zh->errString =
2224                     nmem_strdup (stream, zs->u.simple->u.resultSetId);
2225                 return 0;
2226             }
2227             else
2228                 rset_dup(r);
2229         }
2230         else
2231         {
2232             zh->errCode = 3;
2233             return 0;
2234         }
2235     }
2236     else
2237     {
2238         zh->errCode = 3;
2239         return 0;
2240     }
2241     return r;
2242 }
2243
2244
2245 RSET rpn_search(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
2246                 Z_RPNQuery *rpn, int num_bases, char **basenames,
2247                 const char *setname,
2248                 ZebraSet sset)
2249 {
2250     RSET rset;
2251     oident *attrset;
2252     oid_value attributeSet;
2253     Z_SortKeySpecList *sort_sequence;
2254     int sort_status, i;
2255
2256     zh->errCode = 0;
2257     zh->errString = NULL;
2258     zh->hits = 0;
2259
2260     sort_sequence = (Z_SortKeySpecList *)
2261         nmem_malloc(nmem, sizeof(*sort_sequence));
2262     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
2263     sort_sequence->specs = (Z_SortKeySpec **)
2264         nmem_malloc(nmem, sort_sequence->num_specs *
2265                      sizeof(*sort_sequence->specs));
2266     for (i = 0; i<sort_sequence->num_specs; i++)
2267         sort_sequence->specs[i] = 0;
2268
2269     attrset = oid_getentbyoid (rpn->attributeSetId);
2270     attributeSet = attrset->value;
2271     rset = rpn_search_structure (zh, rpn->RPNStructure, attributeSet,
2272                                  nmem, rset_nmem,
2273                                  sort_sequence, num_bases, basenames);
2274     if (!rset)
2275         return 0;
2276
2277     if (zh->errCode)
2278         yaz_log(YLOG_DEBUG, "search error: %d", zh->errCode);
2279
2280     for (i = 0; sort_sequence->specs[i]; i++)
2281         ;
2282     sort_sequence->num_specs = i;
2283     if (!i)
2284         resultSetRank (zh, sset, rset, rset_nmem);
2285     else
2286     {
2287         yaz_log(YLOG_DEBUG, "resultSetSortSingle in rpn_search");
2288         resultSetSortSingle (zh, nmem, sset, rset,
2289                              sort_sequence, &sort_status);
2290         if (zh->errCode)
2291         {
2292             yaz_log(YLOG_DEBUG, "resultSetSortSingle status = %d", zh->errCode);
2293         }
2294     }
2295     return rset;
2296 }
2297
2298 struct scan_info_entry {
2299     char *term;
2300     ISAMC_P isam_p;
2301 };
2302
2303 struct scan_info {
2304     struct scan_info_entry *list;
2305     ODR odr;
2306     int before, after;
2307     char prefix[20];
2308 };
2309
2310 static int scan_handle (char *name, const char *info, int pos, void *client)
2311 {
2312     int len_prefix, idx;
2313     struct scan_info *scan_info = (struct scan_info *) client;
2314
2315     len_prefix = strlen(scan_info->prefix);
2316     if (memcmp (name, scan_info->prefix, len_prefix))
2317         return 1;
2318     if (pos > 0)        idx = scan_info->after - pos + scan_info->before;
2319     else
2320         idx = - pos - 1;
2321     scan_info->list[idx].term = (char *)
2322         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2323     strcpy(scan_info->list[idx].term, name + len_prefix);
2324     assert (*info == sizeof(ISAMC_P));
2325     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAMC_P));
2326     return 0;
2327 }
2328
2329 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2330                                char **dst, const char *src)
2331 {
2332     char term_src[IT_MAX_WORD];
2333     char term_dst[IT_MAX_WORD];
2334
2335     term_untrans (zh, reg_type, term_src, src);
2336
2337     if (zh->iconv_from_utf8 != 0)
2338     {
2339         int len;
2340         char *inbuf = term_src;
2341         size_t inleft = strlen(term_src);
2342         char *outbuf = term_dst;
2343         size_t outleft = sizeof(term_dst)-1;
2344         size_t ret;
2345
2346         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2347                          &outbuf, &outleft);
2348         if (ret == (size_t)(-1))
2349             len = 0;
2350         else
2351             len = outbuf - term_dst;
2352         *dst = nmem_malloc(stream, len + 1);
2353         if (len > 0)
2354             memcpy (*dst, term_dst, len);
2355         (*dst)[len] = '\0';
2356     }
2357     else
2358         *dst = nmem_strdup(stream, term_src);
2359 }
2360
2361 static void count_set (RSET r, int *count)
2362 {
2363     zint psysno = 0;
2364     int kno = 0;
2365     struct it_key key;
2366     RSFD rfd;
2367
2368     yaz_log(YLOG_DEBUG, "count_set");
2369
2370     *count = 0;
2371     rfd = rset_open (r, RSETF_READ);
2372     while (rset_read (rfd, &key,0 /* never mind terms */))
2373     {
2374         if (key.mem[0] != psysno)
2375         {
2376             psysno = key.mem[0];
2377             (*count)++;
2378         }
2379         kno++;
2380     }
2381     rset_close (rfd);
2382     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2383 }
2384
2385 void rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2386                oid_value attributeset,
2387                int num_bases, char **basenames,
2388                int *position, int *num_entries, ZebraScanEntry **list,
2389                int *is_partial, RSET limit_set, int return_zero)
2390 {
2391     int i;
2392     int pos = *position;
2393     int num = *num_entries;
2394     int before;
2395     int after;
2396     int base_no;
2397     char termz[IT_MAX_WORD+20];
2398     AttrType use;
2399     int use_value;
2400     const char *use_string = 0;
2401     struct scan_info *scan_info_array;
2402     ZebraScanEntry *glist;
2403     int ords[32], ord_no = 0;
2404     int ptr[32];
2405
2406     int bases_ok = 0;     /* no of databases with OK attribute */
2407     int errCode = 0;      /* err code (if any is not OK) */
2408     char *errString = 0;  /* addinfo */
2409
2410     unsigned reg_id;
2411     char *search_type = NULL;
2412     char rank_type[128];
2413     int complete_flag;
2414     int sort_flag;
2415     NMEM rset_nmem = NULL;
2416
2417     *list = 0;
2418
2419     if (attributeset == VAL_NONE)
2420         attributeset = VAL_BIB1;
2421
2422     if (!limit_set)
2423     {
2424         AttrType termset;
2425         int termset_value_numeric;
2426         const char *termset_value_string;
2427         attr_init (&termset, zapt, 8);
2428         termset_value_numeric =
2429             attr_find_ex (&termset, NULL, &termset_value_string);
2430         if (termset_value_numeric != -1)
2431         {
2432             char resname[32];
2433             const char *termset_name = 0;
2434
2435             if (termset_value_numeric != -2)
2436             {
2437
2438                 sprintf (resname, "%d", termset_value_numeric);
2439                 termset_name = resname;
2440             }
2441             else
2442                 termset_name = termset_value_string;
2443
2444             limit_set = resultSetRef (zh, termset_name);
2445         }
2446     }
2447
2448     yaz_log (YLOG_DEBUG, "position = %d, num = %d set=%d",
2449              pos, num, attributeset);
2450
2451     attr_init (&use, zapt, 1);
2452     use_value = attr_find_ex (&use, &attributeset, &use_string);
2453
2454     if (zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2455                          rank_type, &complete_flag, &sort_flag))
2456     {
2457         *num_entries = 0;
2458         zh->errCode = 113;
2459         return ;
2460     }
2461     yaz_log (YLOG_DEBUG, "use_value = %d", use_value);
2462
2463     if (use_value == -1)
2464         use_value = 1016;
2465     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2466     {
2467         int r;
2468         attent attp;
2469         data1_local_attribute *local_attr;
2470
2471         if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2472                                 use_string)))
2473         {
2474             yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2475                   attributeset, use_value);
2476             if (r == -1)
2477             {
2478                 char val_str[32];
2479                 sprintf (val_str, "%d", use_value);
2480                 errCode = 114;
2481                 errString = odr_strdup (stream, val_str);
2482             }
2483             else
2484                 errCode = 121;
2485             continue;
2486         }
2487         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2488         {
2489             zh->errString = basenames[base_no];
2490             zh->errCode = 109; /* Database unavailable */
2491             *num_entries = 0;
2492             return;
2493         }
2494         bases_ok++;
2495         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2496              local_attr = local_attr->next)
2497         {
2498             int ord;
2499
2500             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
2501                                          local_attr->local);
2502             if (ord > 0)
2503                 ords[ord_no++] = ord;
2504         }
2505     }
2506     if (!bases_ok && errCode)
2507     {
2508         zh->errCode = errCode;
2509         zh->errString = errString;
2510         *num_entries = 0;
2511     }
2512     if (ord_no == 0)
2513     {
2514         *num_entries = 0;
2515         return;
2516     }
2517     /* prepare dictionary scanning */
2518     before = pos-1;
2519     after = 1+num-pos;
2520     scan_info_array = (struct scan_info *)
2521         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2522     for (i = 0; i < ord_no; i++)
2523     {
2524         int j, prefix_len = 0;
2525         int before_tmp = before, after_tmp = after;
2526         struct scan_info *scan_info = scan_info_array + i;
2527         struct rpn_char_map_info rcmi;
2528
2529         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2530
2531         scan_info->before = before;
2532         scan_info->after = after;
2533         scan_info->odr = stream;
2534
2535         scan_info->list = (struct scan_info_entry *)
2536             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2537         for (j = 0; j<before+after; j++)
2538             scan_info->list[j].term = NULL;
2539
2540         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2541         termz[prefix_len++] = reg_id;
2542         termz[prefix_len] = 0;
2543         strcpy(scan_info->prefix, termz);
2544
2545         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id))
2546             return ;
2547
2548         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2549                   scan_info, scan_handle);
2550     }
2551     glist = (ZebraScanEntry *)
2552         odr_malloc(stream, (before+after)*sizeof(*glist));
2553
2554     rset_nmem = nmem_create();
2555
2556     /* consider terms after main term */
2557     for (i = 0; i < ord_no; i++)
2558         ptr[i] = before;
2559
2560     *is_partial = 0;
2561     for (i = 0; i<after; i++)
2562     {
2563         int j, j0 = -1;
2564         const char *mterm = NULL;
2565         const char *tst;
2566         RSET rset;
2567
2568         for (j = 0; j < ord_no; j++)
2569         {
2570             if (ptr[j] < before+after &&
2571                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2572                 (!mterm || strcmp (tst, mterm) < 0))
2573             {
2574                 j0 = j;
2575                 mterm = tst;
2576             }
2577         }
2578         if (j0 == -1)
2579             break;
2580         scan_term_untrans (zh, stream->mem, reg_id,
2581                            &glist[i+before].term, mterm);
2582         rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2583                           glist[i+before].term, strlen(glist[i+before].term),
2584                           NULL, 0, zapt->term->which, rset_nmem,
2585                           key_it_ctrl,key_it_ctrl->scope);
2586         ptr[j0]++;
2587         for (j = j0+1; j<ord_no; j++)
2588         {
2589             if (ptr[j] < before+after &&
2590                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2591                 !strcmp (tst, mterm))
2592             {
2593                 RSET rsets[2];
2594
2595                 rsets[0] = rset;
2596                 rsets[1] =
2597                     rset_trunc(zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2598                                glist[i+before].term,
2599                                strlen(glist[i+before].term), NULL, 0,
2600                                zapt->term->which,rset_nmem,
2601                                key_it_ctrl, key_it_ctrl->scope);
2602                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2603                                          2, key_it_ctrl->scope, rsets);
2604                 ptr[j]++;
2605             }
2606         }
2607         if (limit_set)
2608         {
2609             RSET rsets[2];
2610             rsets[0] = rset;
2611             rsets[1] = rset_dup(limit_set);
2612
2613             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2614                                       key_it_ctrl->scope, 2, rsets);
2615         }
2616         count_set(rset, &glist[i+before].occurrences);
2617         rset_delete(rset);
2618     }
2619     if (i < after)
2620     {
2621         *num_entries -= (after-i);
2622         *is_partial = 1;
2623     }
2624
2625     /* consider terms before main term */
2626     for (i = 0; i<ord_no; i++)
2627         ptr[i] = 0;
2628
2629     for (i = 0; i<before; i++)
2630     {
2631         int j, j0 = -1;
2632         const char *mterm = NULL;
2633         const char *tst;
2634         RSET rset;
2635
2636         for (j = 0; j <ord_no; j++)
2637         {
2638             if (ptr[j] < before &&
2639                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2640                 (!mterm || strcmp (tst, mterm) > 0))
2641             {
2642                 j0 = j;
2643                     mterm = tst;
2644             }
2645         }
2646         if (j0 == -1)
2647             break;
2648
2649         scan_term_untrans (zh, stream->mem, reg_id,
2650                            &glist[before-1-i].term, mterm);
2651
2652         rset = rset_trunc
2653             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2654              glist[before-1-i].term, strlen(glist[before-1-i].term),
2655              NULL, 0, zapt->term->which,rset_nmem,
2656              key_it_ctrl,key_it_ctrl->scope);
2657
2658         ptr[j0]++;
2659
2660         for (j = j0+1; j<ord_no; j++)
2661         {
2662             if (ptr[j] < before &&
2663                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2664                 !strcmp (tst, mterm))
2665             {
2666                 RSET rsets[2];
2667
2668                 rsets[0] = rset;
2669                 rsets[1] = rset_trunc(
2670                     zh,
2671                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2672                     glist[before-1-i].term,
2673                     strlen(glist[before-1-i].term), NULL, 0,
2674                     zapt->term->which, rset_nmem,
2675                     key_it_ctrl, key_it_ctrl->scope);
2676                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2677                                          2, key_it_ctrl->scope, rsets);
2678
2679                 ptr[j]++;
2680             }
2681         }
2682         if (limit_set)
2683         {
2684             RSET rsets[2];
2685             rsets[0] = rset;
2686             rsets[1] = rset_dup(limit_set);
2687
2688             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2689                                       key_it_ctrl->scope, 2, rsets);
2690         }
2691         count_set (rset, &glist[before-1-i].occurrences);
2692         rset_delete (rset);
2693     }
2694     i = before-i;
2695     if (i)
2696     {
2697         *is_partial = 1;
2698         *position -= i;
2699         *num_entries -= i;
2700     }
2701
2702     nmem_destroy(rset_nmem);
2703     *list = glist + i;               /* list is set to first 'real' entry */
2704
2705     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2706             *position, *num_entries);
2707     if (zh->errCode)
2708         yaz_log(YLOG_DEBUG, "scan error: %d", zh->errCode);
2709 }
2710