zrpn.c

   1 /* $Id: zrpn.c,v 1.164 2004-12-10 12:37:07 heikki Exp $
   2    Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
   3    Index Data Aps
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39
  40 static const struct key_control it_ctrl =
  41 {
  42     sizeof(struct it_key),
  43     2, /* we have sysnos and seqnos in this key, nothing more */
  44     key_compare_it,
  45     key_logdump_txt,   /* FIXME  - clean up these functions */
  46     key_get_seq,
  47 };
  48
  49
  50 const struct key_control *key_it_ctrl = &it_ctrl;
  51
  52 struct rpn_char_map_info
  53 {
  54     ZebraMaps zm;
  55     int reg_type;
  56 };
  57
  58 typedef struct
  59 {
  60     int type;
  61     int major;
  62     int minor;
  63     Z_AttributesPlusTerm *zapt;
  64 } AttrType;
  65
  66
  67 static int log_level_set=0;
  68 static int log_level_rpn=0;
  69
  70 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  71 {
  72     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  73     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  74 #if 0
  75     if (out && *out)
  76     {
  77         const char *outp = *out;
  78         yaz_log(YLOG_LOG, "---");
  79         while (*outp)
  80         {
  81             yaz_log(YLOG_LOG, "%02X", *outp);
  82             outp++;
  83         }
  84     }
  85 #endif
  86     return out;
  87 }
  88
  89 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  90                                   struct rpn_char_map_info *map_info)
  91 {
  92     map_info->zm = reg->zebra_maps;
  93     map_info->reg_type = reg_type;
  94     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  95 }
  96
  97 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  98                          const char **string_value)
  99 {
 100     int num_attributes;
 101
 102     num_attributes = src->zapt->attributes->num_attributes;
 103     while (src->major < num_attributes)
 104     {
 105         Z_AttributeElement *element;
 106
 107         element = src->zapt->attributes->attributes[src->major];
 108         if (src->type == *element->attributeType)
 109         {
 110             switch (element->which)
 111             {
 112             case Z_AttributeValue_numeric:
 113                 ++(src->major);
 114                 if (element->attributeSet && attributeSetP)
 115                 {
 116                     oident *attrset;
 117
 118                     attrset = oid_getentbyoid(element->attributeSet);
 119                     *attributeSetP = attrset->value;
 120                 }
 121                 return *element->value.numeric;
 122                 break;
 123             case Z_AttributeValue_complex:
 124                 if (src->minor >= element->value.complex->num_list)
 125                     break;
 126                 if (element->attributeSet && attributeSetP)
 127                 {
 128                     oident *attrset;
 129
 130                     attrset = oid_getentbyoid(element->attributeSet);
 131                     *attributeSetP = attrset->value;
 132                 }
 133                 if (element->value.complex->list[src->minor]->which ==
 134                     Z_StringOrNumeric_numeric)
 135                 {
 136                     ++(src->minor);
 137                     return
 138                         *element->value.complex->list[src->minor-1]->u.numeric;
 139                 }
 140                 else if (element->value.complex->list[src->minor]->which ==
 141                          Z_StringOrNumeric_string)
 142                 {
 143                     if (!string_value)
 144                         break;
 145                     ++(src->minor);
 146                     *string_value =
 147                         element->value.complex->list[src->minor-1]->u.string;
 148                     return -2;
 149                 }
 150                 else
 151                     break;
 152             default:
 153                 assert(0);
 154             }
 155         }
 156         ++(src->major);
 157     }
 158     return -1;
 159 }
 160
 161 static int attr_find(AttrType *src, oid_value *attributeSetP)
 162 {
 163     return attr_find_ex(src, attributeSetP, 0);
 164 }
 165
 166 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 167                        int type)
 168 {
 169     src->zapt = zapt;
 170     src->type = type;
 171     src->major = 0;
 172     src->minor = 0;
 173 }
 174
 175 #define TERM_COUNT
 176
 177 struct grep_info {
 178 #ifdef TERM_COUNT
 179     int *term_no;
 180 #endif
 181     ISAMC_P *isam_p_buf;
 182     int isam_p_size;
 183     int isam_p_indx;
 184     ZebraHandle zh;
 185     int reg_type;
 186     ZebraSet termset;
 187 };
 188
 189 static void term_untrans(ZebraHandle zh, int reg_type,
 190                            char *dst, const char *src)
 191 {
 192     int len = 0;
 193     while (*src)
 194     {
 195         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 196                                             reg_type, &src);
 197         if (!cp && len < IT_MAX_WORD-1)
 198             dst[len++] = *src++;
 199         else
 200             while (*cp && len < IT_MAX_WORD-1)
 201                 dst[len++] = *cp++;
 202     }
 203     dst[len] = '\0';
 204 }
 205
 206 static void add_isam_p(const char *name, const char *info,
 207                         struct grep_info *p)
 208 {
 209     if (!log_level_set)
 210     {
 211         log_level_rpn = yaz_log_module_level("rpn");
 212         log_level_set=1;
 213     }
 214     if (p->isam_p_indx == p->isam_p_size)
 215     {
 216         ISAMC_P *new_isam_p_buf;
 217 #ifdef TERM_COUNT
 218         int *new_term_no;
 219 #endif
 220         p->isam_p_size = 2*p->isam_p_size + 100;
 221         new_isam_p_buf = (ISAMC_P *) xmalloc(sizeof(*new_isam_p_buf) *
 222                                              p->isam_p_size);
 223         if (p->isam_p_buf)
 224         {
 225             memcpy(new_isam_p_buf, p->isam_p_buf,
 226                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 227             xfree(p->isam_p_buf);
 228         }
 229         p->isam_p_buf = new_isam_p_buf;
 230
 231 #ifdef TERM_COUNT
 232         new_term_no = (int *) xmalloc(sizeof(*new_term_no) *
 233                                        p->isam_p_size);
 234         if (p->term_no)
 235         {
 236             memcpy(new_term_no, p->isam_p_buf,
 237                     p->isam_p_indx * sizeof(*p->term_no));
 238             xfree(p->term_no);
 239         }
 240         p->term_no = new_term_no;
 241 #endif
 242     }
 243     assert(*info == sizeof(*p->isam_p_buf));
 244     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 245
 246 #if 1
 247     if (p->termset)
 248     {
 249         const char *db;
 250         int set, use;
 251         char term_tmp[IT_MAX_WORD];
 252         int su_code = 0;
 253         int len = key_SU_decode (&su_code, name);
 254
 255         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 256         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 257         zebraExplain_lookup_ord (p->zh->reg->zei,
 258                                  su_code, &db, &set, &use);
 259         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 260
 261         resultSetAddTerm(p->zh, p->termset, name[len], db,
 262                          set, use, term_tmp);
 263     }
 264 #endif
 265     (p->isam_p_indx)++;
 266 }
 267
 268 static int grep_handle(char *name, const char *info, void *p)
 269 {
 270     add_isam_p(name, info, (struct grep_info *) p);
 271     return 0;
 272 }
 273
 274 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 275                      const char *ct1, const char *ct2, int first)
 276 {
 277     const char *s1, *s0 = *src;
 278     const char **map;
 279
 280     /* skip white space */
 281     while (*s0)
 282     {
 283         if (ct1 && strchr(ct1, *s0))
 284             break;
 285         if (ct2 && strchr(ct2, *s0))
 286             break;
 287         s1 = s0;
 288         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 289         if (**map != *CHR_SPACE)
 290             break;
 291         s0 = s1;
 292     }
 293     *src = s0;
 294     return *s0;
 295 }
 296
 297 #define REGEX_CHARS " []()|.*+?!"
 298
 299 /* term_100: handle term, where trunc=none(no operators at all) */
 300 static int term_100(ZebraMaps zebra_maps, int reg_type,
 301                      const char **src, char *dst, int space_split,
 302                      char *dst_term)
 303 {
 304     const char *s0, *s1;
 305     const char **map;
 306     int i = 0;
 307     int j = 0;
 308
 309     const char *space_start = 0;
 310     const char *space_end = 0;
 311
 312     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 313         return 0;
 314     s0 = *src;
 315     while (*s0)
 316     {
 317         s1 = s0;
 318         map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 319         if (space_split)
 320         {
 321             if (**map == *CHR_SPACE)
 322                 break;
 323         }
 324         else  /* complete subfield only. */
 325         {
 326             if (**map == *CHR_SPACE)
 327             {   /* save space mapping for later  .. */
 328                 space_start = s1;
 329                 space_end = s0;
 330                 continue;
 331             }
 332             else if (space_start)
 333             {   /* reload last space */
 334                 while (space_start < space_end)
 335                 {
 336                     if (strchr(REGEX_CHARS, *space_start))
 337                         dst[i++] = '\\';
 338                     dst_term[j++] = *space_start;
 339                     dst[i++] = *space_start++;
 340                 }
 341                 /* and reset */
 342                 space_start = space_end = 0;
 343             }
 344         }
 345         /* add non-space char */
 346         while (s1 < s0)
 347         {
 348             if (strchr(REGEX_CHARS, *s1))
 349                 dst[i++] = '\\';
 350             dst_term[j++] = *s1;
 351             dst[i++] = *s1++;
 352         }
 353     }
 354     dst[i] = '\0';
 355     dst_term[j] = '\0';
 356     *src = s0;
 357     return i;
 358 }
 359
 360 /* term_101: handle term, where trunc=Process # */
 361 static int term_101(ZebraMaps zebra_maps, int reg_type,
 362                      const char **src, char *dst, int space_split,
 363                      char *dst_term)
 364 {
 365     const char *s0, *s1;
 366     const char **map;
 367     int i = 0;
 368     int j = 0;
 369
 370     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 371         return 0;
 372     s0 = *src;
 373     while (*s0)
 374     {
 375         if (*s0 == '#')
 376         {
 377             dst[i++] = '.';
 378             dst[i++] = '*';
 379             dst_term[j++] = *s0++;
 380         }
 381         else
 382         {
 383             s1 = s0;
 384             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 385             if (space_split && **map == *CHR_SPACE)
 386                 break;
 387             while (s1 < s0)
 388             {
 389                 if (strchr(REGEX_CHARS, *s1))
 390                     dst[i++] = '\\';
 391                 dst_term[j++] = *s1;
 392                 dst[i++] = *s1++;
 393             }
 394         }
 395     }
 396     dst[i] = '\0';
 397     dst_term[j++] = '\0';
 398     *src = s0;
 399     return i;
 400 }
 401
 402 /* term_103: handle term, where trunc=re-2 (regular expressions) */
 403 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 404                      char *dst, int *errors, int space_split,
 405                      char *dst_term)
 406 {
 407     int i = 0;
 408     int j = 0;
 409     const char *s0, *s1;
 410     const char **map;
 411
 412     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 413         return 0;
 414     s0 = *src;
 415     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 416         isdigit(s0[1]))
 417     {
 418         *errors = s0[1] - '0';
 419         s0 += 3;
 420         if (*errors > 3)
 421             *errors = 3;
 422     }
 423     while (*s0)
 424     {
 425         if (strchr("^\\()[].*+?|-", *s0))
 426         {
 427             dst_term[j++] = *s0;
 428             dst[i++] = *s0++;
 429         }
 430         else
 431         {
 432             s1 = s0;
 433             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 434             if (**map == *CHR_SPACE)
 435                 break;
 436             while (s1 < s0)
 437             {
 438                 if (strchr(REGEX_CHARS, *s1))
 439                     dst[i++] = '\\';
 440                 dst_term[j++] = *s1;
 441                 dst[i++] = *s1++;
 442             }
 443         }
 444     }
 445     dst[i] = '\0';
 446     dst_term[j] = '\0';
 447     *src = s0;
 448     return i;
 449 }
 450
 451 /* term_103: handle term, where trunc=re-1 (regular expressions) */
 452 static int term_102 (ZebraMaps zebra_maps, int reg_type, const char **src,
 453                      char *dst, int space_split, char *dst_term)
 454 {
 455     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 456                      dst_term);
 457 }
 458
 459
 460 /* term_104: handle term, where trunc=Process # and ! */
 461 static int term_104(ZebraMaps zebra_maps, int reg_type,
 462                      const char **src, char *dst, int space_split,
 463                      char *dst_term)
 464 {
 465     const char *s0, *s1;
 466     const char **map;
 467     int i = 0;
 468     int j = 0;
 469
 470     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 471         return 0;
 472     s0 = *src;
 473     while (*s0)
 474     {
 475         if (*s0 == '?')
 476         {
 477             dst_term[j++] = *s0++;
 478             if (*s0 >= '0' && *s0 <= '9')
 479             {
 480                 int limit = 0;
 481                 while (*s0 >= '0' && *s0 <= '9')
 482                 {
 483                     limit = limit * 10 + (*s0 - '0');
 484                     dst_term[j++] = *s0++;
 485                 }
 486                 if (limit > 20)
 487                     limit = 20;
 488                 while (--limit >= 0)
 489                 {
 490                     dst[i++] = '.';
 491                     dst[i++] = '?';
 492                 }
 493             }
 494             else
 495             {
 496                 dst[i++] = '.';
 497                 dst[i++] = '*';
 498             }
 499         }
 500         else if (*s0 == '*')
 501         {
 502             dst[i++] = '.';
 503             dst[i++] = '*';
 504             dst_term[j++] = *s0++;
 505         }
 506         else if (*s0 == '#')
 507         {
 508             dst[i++] = '.';
 509             dst_term[j++] = *s0++;
 510         }
 511         {
 512             s1 = s0;
 513             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 514             if (space_split && **map == *CHR_SPACE)
 515                 break;
 516             while (s1 < s0)
 517             {
 518                 if (strchr(REGEX_CHARS, *s1))
 519                     dst[i++] = '\\';
 520                 dst_term[j++] = *s1;
 521                 dst[i++] = *s1++;
 522             }
 523         }
 524     }
 525     dst[i] = '\0';
 526     dst_term[j++] = '\0';
 527     *src = s0;
 528     return i;
 529 }
 530
 531 /* term_105/106: handle term, where trunc=Process * and ! and right trunc */
 532 static int term_105 (ZebraMaps zebra_maps, int reg_type,
 533                      const char **src, char *dst, int space_split,
 534                      char *dst_term, int right_truncate)
 535 {
 536     const char *s0, *s1;
 537     const char **map;
 538     int i = 0;
 539     int j = 0;
 540
 541     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 542         return 0;
 543     s0 = *src;
 544     while (*s0)
 545     {
 546         if (*s0 == '*')
 547         {
 548             dst[i++] = '.';
 549             dst[i++] = '*';
 550             dst_term[j++] = *s0++;
 551         }
 552         else if (*s0 == '!')
 553         {
 554             dst[i++] = '.';
 555             dst_term[j++] = *s0++;
 556         }
 557         {
 558             s1 = s0;
 559             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 560             if (space_split && **map == *CHR_SPACE)
 561                 break;
 562             while (s1 < s0)
 563             {
 564                 if (strchr(REGEX_CHARS, *s1))
 565                     dst[i++] = '\\';
 566                 dst_term[j++] = *s1;
 567                 dst[i++] = *s1++;
 568             }
 569         }
 570     }
 571     if (right_truncate)
 572     {
 573         dst[i++] = '.';
 574         dst[i++] = '*';
 575     }
 576     dst[i] = '\0';
 577
 578     dst_term[j++] = '\0';
 579     *src = s0;
 580     return i;
 581 }
 582
 583
 584 /* gen_regular_rel - generate regular expression from relation
 585  *  val:     border value (inclusive)
 586  *  islt:    1 if <=; 0 if >=.
 587  */
 588 static void gen_regular_rel(char *dst, int val, int islt)
 589 {
 590     int dst_p;
 591     int w, d, i;
 592     int pos = 0;
 593     char numstr[20];
 594
 595     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 596     if (val >= 0)
 597     {
 598         if (islt)
 599             strcpy(dst, "(-[0-9]+|(");
 600         else
 601             strcpy(dst, "((");
 602     }
 603     else
 604     {
 605         if (!islt)
 606         {
 607             strcpy(dst, "([0-9]+|-(");
 608             dst_p = strlen(dst);
 609             islt = 1;
 610         }
 611         else
 612         {
 613             strcpy(dst, "(-(");
 614             islt = 0;
 615         }
 616         val = -val;
 617     }
 618     dst_p = strlen(dst);
 619     sprintf(numstr, "%d", val);
 620     for (w = strlen(numstr); --w >= 0; pos++)
 621     {
 622         d = numstr[w];
 623         if (pos > 0)
 624         {
 625             if (islt)
 626             {
 627                 if (d == '0')
 628                     continue;
 629                 d--;
 630             }
 631             else
 632             {
 633                 if (d == '9')
 634                     continue;
 635                 d++;
 636             }
 637         }
 638
 639         strcpy(dst + dst_p, numstr);
 640         dst_p = strlen(dst) - pos - 1;
 641
 642         if (islt)
 643         {
 644             if (d != '0')
 645             {
 646                 dst[dst_p++] = '[';
 647                 dst[dst_p++] = '0';
 648                 dst[dst_p++] = '-';
 649                 dst[dst_p++] = d;
 650                 dst[dst_p++] = ']';
 651             }
 652             else
 653                 dst[dst_p++] = d;
 654         }
 655         else
 656         {
 657             if (d != '9')
 658             {
 659                 dst[dst_p++] = '[';
 660                 dst[dst_p++] = d;
 661                 dst[dst_p++] = '-';
 662                 dst[dst_p++] = '9';
 663                 dst[dst_p++] = ']';
 664             }
 665             else
 666                 dst[dst_p++] = d;
 667         }
 668         for (i = 0; i<pos; i++)
 669         {
 670             dst[dst_p++] = '[';
 671             dst[dst_p++] = '0';
 672             dst[dst_p++] = '-';
 673             dst[dst_p++] = '9';
 674             dst[dst_p++] = ']';
 675         }
 676         dst[dst_p++] = '|';
 677     }
 678     dst[dst_p] = '\0';
 679     if (islt)
 680     {
 681         /* match everything less than 10^(pos-1) */
 682         strcat(dst, "0*");
 683         for (i=1; i<pos; i++)
 684             strcat(dst, "[0-9]?");
 685     }
 686     else
 687     {
 688         /* match everything greater than 10^pos */
 689         for (i = 0; i <= pos; i++)
 690             strcat(dst, "[0-9]");
 691         strcat(dst, "[0-9]*");
 692     }
 693     strcat(dst, "))");
 694 }
 695
 696 void string_rel_add_char(char **term_p, const char *src, int *indx)
 697 {
 698     if (src[*indx] == '\\')
 699         *(*term_p)++ = src[(*indx)++];
 700     *(*term_p)++ = src[(*indx)++];
 701 }
 702
 703 /*
 704  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 705  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 706  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 707  *              ([^-a].*|a[^-b].*|ab[c-].*)
 708  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 709  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 710  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 711  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 712  */
 713 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 714                             const char **term_sub, char *term_dict,
 715                             oid_value attributeSet,
 716                             int reg_type, int space_split, char *term_dst)
 717 {
 718     AttrType relation;
 719     int relation_value;
 720     int i;
 721     char *term_tmp = term_dict + strlen(term_dict);
 722     char term_component[2*IT_MAX_WORD+20];
 723
 724     attr_init(&relation, zapt, 2);
 725     relation_value = attr_find(&relation, NULL);
 726
 727     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 728     switch (relation_value)
 729     {
 730     case 1:
 731         if (!term_100 (zh->reg->zebra_maps, reg_type,
 732                        term_sub, term_component,
 733                        space_split, term_dst))
 734             return 0;
 735         yaz_log(log_level_rpn, "Relation <");
 736
 737         *term_tmp++ = '(';
 738         for (i = 0; term_component[i]; )
 739         {
 740             int j = 0;
 741
 742             if (i)
 743                 *term_tmp++ = '|';
 744             while (j < i)
 745                 string_rel_add_char (&term_tmp, term_component, &j);
 746
 747             *term_tmp++ = '[';
 748
 749             *term_tmp++ = '^';
 750             string_rel_add_char (&term_tmp, term_component, &i);
 751             *term_tmp++ = '-';
 752
 753             *term_tmp++ = ']';
 754             *term_tmp++ = '.';
 755             *term_tmp++ = '*';
 756
 757             if ((term_tmp - term_dict) > IT_MAX_WORD)
 758                 break;
 759         }
 760         *term_tmp++ = ')';
 761         *term_tmp = '\0';
 762         break;
 763     case 2:
 764         if (!term_100 (zh->reg->zebra_maps, reg_type,
 765                        term_sub, term_component,
 766                        space_split, term_dst))
 767             return 0;
 768         yaz_log(log_level_rpn, "Relation <=");
 769
 770         *term_tmp++ = '(';
 771         for (i = 0; term_component[i]; )
 772         {
 773             int j = 0;
 774
 775             while (j < i)
 776                 string_rel_add_char (&term_tmp, term_component, &j);
 777             *term_tmp++ = '[';
 778
 779             *term_tmp++ = '^';
 780             string_rel_add_char (&term_tmp, term_component, &i);
 781             *term_tmp++ = '-';
 782
 783             *term_tmp++ = ']';
 784             *term_tmp++ = '.';
 785             *term_tmp++ = '*';
 786
 787             *term_tmp++ = '|';
 788
 789             if ((term_tmp - term_dict) > IT_MAX_WORD)
 790                 break;
 791         }
 792         for (i = 0; term_component[i]; )
 793             string_rel_add_char (&term_tmp, term_component, &i);
 794         *term_tmp++ = ')';
 795         *term_tmp = '\0';
 796         break;
 797     case 5:
 798         if (!term_100 (zh->reg->zebra_maps, reg_type,
 799                        term_sub, term_component, space_split, term_dst))
 800             return 0;
 801         yaz_log(log_level_rpn, "Relation >");
 802
 803         *term_tmp++ = '(';
 804         for (i = 0; term_component[i];)
 805         {
 806             int j = 0;
 807
 808             while (j < i)
 809                 string_rel_add_char (&term_tmp, term_component, &j);
 810             *term_tmp++ = '[';
 811
 812             *term_tmp++ = '^';
 813             *term_tmp++ = '-';
 814             string_rel_add_char (&term_tmp, term_component, &i);
 815
 816             *term_tmp++ = ']';
 817             *term_tmp++ = '.';
 818             *term_tmp++ = '*';
 819
 820             *term_tmp++ = '|';
 821
 822             if ((term_tmp - term_dict) > IT_MAX_WORD)
 823                 break;
 824         }
 825         for (i = 0; term_component[i];)
 826             string_rel_add_char (&term_tmp, term_component, &i);
 827         *term_tmp++ = '.';
 828         *term_tmp++ = '+';
 829         *term_tmp++ = ')';
 830         *term_tmp = '\0';
 831         break;
 832     case 4:
 833         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 834                        term_component, space_split, term_dst))
 835             return 0;
 836         yaz_log(log_level_rpn, "Relation >=");
 837
 838         *term_tmp++ = '(';
 839         for (i = 0; term_component[i];)
 840         {
 841             int j = 0;
 842
 843             if (i)
 844                 *term_tmp++ = '|';
 845             while (j < i)
 846                 string_rel_add_char (&term_tmp, term_component, &j);
 847             *term_tmp++ = '[';
 848
 849             if (term_component[i+1])
 850             {
 851                 *term_tmp++ = '^';
 852                 *term_tmp++ = '-';
 853                 string_rel_add_char (&term_tmp, term_component, &i);
 854             }
 855             else
 856             {
 857                 string_rel_add_char (&term_tmp, term_component, &i);
 858                 *term_tmp++ = '-';
 859             }
 860             *term_tmp++ = ']';
 861             *term_tmp++ = '.';
 862             *term_tmp++ = '*';
 863
 864             if ((term_tmp - term_dict) > IT_MAX_WORD)
 865                 break;
 866         }
 867         *term_tmp++ = ')';
 868         *term_tmp = '\0';
 869         break;
 870     case 3:
 871     default:
 872         yaz_log(log_level_rpn, "Relation =");
 873         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 874                        term_component, space_split, term_dst))
 875             return 0;
 876         strcat(term_tmp, "(");
 877         strcat(term_tmp, term_component);
 878         strcat(term_tmp, ")");
 879     }
 880     return 1;
 881 }
 882
 883 static int string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 884                         const char **term_sub,
 885                         oid_value attributeSet, NMEM stream,
 886                         struct grep_info *grep_info,
 887                         int reg_type, int complete_flag,
 888                         int num_bases, char **basenames,
 889                         char *term_dst, int xpath_use);
 890
 891 static RSET term_trunc(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 892                         const char **term_sub,
 893                         oid_value attributeSet, NMEM stream,
 894                         struct grep_info *grep_info,
 895                         int reg_type, int complete_flag,
 896                         int num_bases, char **basenames,
 897                         char *term_dst,
 898                         const char *rank_type, int xpath_use,
 899                         NMEM rset_nmem)
 900 {
 901     int r;
 902     grep_info->isam_p_indx = 0;
 903     r = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 904                      reg_type, complete_flag, num_bases, basenames,
 905                      term_dst, xpath_use);
 906     if (r < 1)
 907         return 0;
 908     yaz_log(log_level_rpn, "term: %s", term_dst);
 909     return rset_trunc(zh, grep_info->isam_p_buf,
 910                        grep_info->isam_p_indx, term_dst,
 911                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 912                        zapt->term->which, rset_nmem,
 913                        key_it_ctrl,key_it_ctrl->scope);
 914 }
 915 static char *nmem_strdup_i(NMEM nmem, int v)
 916 {
 917     char val_str[64];
 918     sprintf (val_str, "%d", v);
 919     return nmem_strdup(nmem, val_str);
 920 }
 921
 922 static int string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 923                        const char **term_sub,
 924                        oid_value attributeSet, NMEM stream,
 925                        struct grep_info *grep_info,
 926                        int reg_type, int complete_flag,
 927                        int num_bases, char **basenames,
 928                        char *term_dst, int xpath_use)
 929 {
 930     char term_dict[2*IT_MAX_WORD+4000];
 931     int j, r, base_no;
 932     AttrType truncation;
 933     int truncation_value;
 934     AttrType use;
 935     int use_value;
 936     const char *use_string = 0;
 937     oid_value curAttributeSet = attributeSet;
 938     const char *termp;
 939     struct rpn_char_map_info rcmi;
 940     int space_split = complete_flag ? 0 : 1;
 941
 942     int bases_ok = 0;     /* no of databases with OK attribute */
 943     int errCode = 0;      /* err code (if any is not OK) */
 944     char *errString = 0;  /* addinfo */
 945
 946     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
 947     attr_init (&use, zapt, 1);
 948     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
 949     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
 950     attr_init (&truncation, zapt, 5);
 951     truncation_value = attr_find (&truncation, NULL);
 952     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
 953
 954     if (use_value == -1)    /* no attribute - assumy "any" */
 955         use_value = 1016;
 956     for (base_no = 0; base_no < num_bases; base_no++)
 957     {
 958         int attr_ok = 0;
 959         int regex_range = 0;
 960         int init_pos = 0;
 961         attent attp;
 962         data1_local_attribute id_xpath_attr;
 963         data1_local_attribute *local_attr;
 964         int max_pos, prefix_len = 0;
 965
 966         termp = *term_sub;
 967
 968         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
 969         {
 970             zh->errCode = 109; /* Database unavailable */
 971             zh->errString = basenames[base_no];
 972             return -1;
 973         }
 974         if (xpath_use > 0 && use_value == -2)
 975         {
 976             use_value = xpath_use;
 977             attp.local_attributes = &id_xpath_attr;
 978             attp.attset_ordinal = VAL_IDXPATH;
 979             id_xpath_attr.next = 0;
 980             id_xpath_attr.local = use_value;
 981         }
 982         else if (curAttributeSet == VAL_IDXPATH)
 983         {
 984             attp.local_attributes = &id_xpath_attr;
 985             attp.attset_ordinal = VAL_IDXPATH;
 986             id_xpath_attr.next = 0;
 987             id_xpath_attr.local = use_value;
 988         }
 989         else
 990         {
 991             if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value,
 992                                             use_string)))
 993             {
 994                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
 995                       curAttributeSet, use_value, r);
 996                 if (r == -1)
 997                 {
 998                     /* set was found, but value wasn't defined */
 999                     errCode = 114;
1000                     if (use_string)
1001                         errString = nmem_strdup(stream, use_string);
1002                     else
1003                         errString = nmem_strdup_i (stream, use_value);
1004                 }
1005                 else
1006                 {
1007                     int oid[OID_SIZE];
1008                     struct oident oident;
1009
1010                     oident.proto = PROTO_Z3950;
1011                     oident.oclass = CLASS_ATTSET;
1012                     oident.value = curAttributeSet;
1013                     oid_ent_to_oid (&oident, oid);
1014
1015                     errCode = 121;
1016                     errString = nmem_strdup (stream, oident.desc);
1017                 }
1018                 continue;
1019             }
1020         }
1021         for (local_attr = attp.local_attributes; local_attr;
1022              local_attr = local_attr->next)
1023         {
1024             int ord;
1025             char ord_buf[32];
1026             int i, ord_len;
1027
1028             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
1029                                          local_attr->local);
1030             if (ord < 0)
1031                 continue;
1032             if (prefix_len)
1033                 term_dict[prefix_len++] = '|';
1034             else
1035                 term_dict[prefix_len++] = '(';
1036
1037             ord_len = key_SU_encode (ord, ord_buf);
1038             for (i = 0; i<ord_len; i++)
1039             {
1040                 term_dict[prefix_len++] = 1;
1041                 term_dict[prefix_len++] = ord_buf[i];
1042             }
1043         }
1044         if (!prefix_len)
1045         {
1046 #if 1
1047             bases_ok++;
1048 #else
1049             errCode = 114;
1050             errString = nmem_strdup_i(stream, use_value);
1051             continue;
1052 #endif
1053         }
1054         else
1055         {
1056             bases_ok++; /* this has OK attributes */
1057             attr_ok = 1;
1058         }
1059
1060         term_dict[prefix_len++] = ')';
1061         term_dict[prefix_len++] = 1;
1062         term_dict[prefix_len++] = reg_type;
1063         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1064         term_dict[prefix_len] = '\0';
1065         j = prefix_len;
1066         switch (truncation_value)
1067         {
1068         case -1:         /* not specified */
1069         case 100:        /* do not truncate */
1070             if (!string_relation (zh, zapt, &termp, term_dict,
1071                                   attributeSet,
1072                                   reg_type, space_split, term_dst))
1073                 return 0;
1074             break;
1075         case 1:          /* right truncation */
1076             term_dict[j++] = '(';
1077             if (!term_100(zh->reg->zebra_maps, reg_type,
1078                           &termp, term_dict + j, space_split, term_dst))
1079                 return 0;
1080             strcat(term_dict, ".*)");
1081             break;
1082         case 2:          /* keft truncation */
1083             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1084             if (!term_100(zh->reg->zebra_maps, reg_type,
1085                           &termp, term_dict + j, space_split, term_dst))
1086                 return 0;
1087             strcat(term_dict, ")");
1088             break;
1089         case 3:          /* left&right truncation */
1090             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1091             if (!term_100(zh->reg->zebra_maps, reg_type,
1092                           &termp, term_dict + j, space_split, term_dst))
1093                 return 0;
1094             strcat(term_dict, ".*)");
1095             break;
1096         case 101:        /* process # in term */
1097             term_dict[j++] = '(';
1098             if (!term_101(zh->reg->zebra_maps, reg_type,
1099                           &termp, term_dict + j, space_split, term_dst))
1100                 return 0;
1101             strcat(term_dict, ")");
1102             break;
1103         case 102:        /* Regexp-1 */
1104             term_dict[j++] = '(';
1105             if (!term_102(zh->reg->zebra_maps, reg_type,
1106                           &termp, term_dict + j, space_split, term_dst))
1107                 return 0;
1108             strcat(term_dict, ")");
1109             break;
1110         case 103:       /* Regexp-2 */
1111             r = 1;
1112             term_dict[j++] = '(';
1113             init_pos = 2;
1114             if (!term_103 (zh->reg->zebra_maps, reg_type,
1115                            &termp, term_dict + j, &regex_range,
1116                            space_split, term_dst))
1117                 return 0;
1118             strcat(term_dict, ")");
1119         case 104:        /* process # and ! in term */
1120             term_dict[j++] = '(';
1121             if (!term_104 (zh->reg->zebra_maps, reg_type,
1122                            &termp, term_dict + j, space_split, term_dst))
1123                 return 0;
1124             strcat(term_dict, ")");
1125             break;
1126         case 105:        /* process * and ! in term */
1127             term_dict[j++] = '(';
1128             if (!term_105 (zh->reg->zebra_maps, reg_type,
1129                            &termp, term_dict + j, space_split, term_dst, 1))
1130                 return 0;
1131             strcat(term_dict, ")");
1132             break;
1133         case 106:        /* process * and ! in term */
1134             term_dict[j++] = '(';
1135             if (!term_105 (zh->reg->zebra_maps, reg_type,
1136                            &termp, term_dict + j, space_split, term_dst, 0))
1137                 return 0;
1138             strcat(term_dict, ")");
1139             break;
1140         default:
1141             zh->errCode = 120;
1142             zh->errString = nmem_strdup_i(stream, truncation_value);
1143             return -1;
1144         }
1145         if (attr_ok)
1146         {
1147             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1148             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1149                                  grep_info, &max_pos, init_pos,
1150                                  grep_handle);
1151             if (r)
1152                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1153         }
1154     }
1155     if (!bases_ok)
1156     {
1157         zh->errCode = errCode;
1158         zh->errString = errString;
1159         return -1;
1160     }
1161     *term_sub = termp;
1162     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1163     return 1;
1164 }
1165
1166
1167 /* convert APT search term to UTF8 */
1168 static int zapt_term_to_utf8 (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1169                               char *termz)
1170 {
1171     size_t sizez;
1172     Z_Term *term = zapt->term;
1173
1174     switch (term->which)
1175     {
1176     case Z_Term_general:
1177         if (zh->iconv_to_utf8 != 0)
1178         {
1179             char *inbuf = term->u.general->buf;
1180             size_t inleft = term->u.general->len;
1181             char *outbuf = termz;
1182             size_t outleft = IT_MAX_WORD-1;
1183             size_t ret;
1184
1185             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1186                         &outbuf, &outleft);
1187             if (ret == (size_t)(-1))
1188             {
1189                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1190                 zh->errCode = 125;
1191                 return -1;
1192             }
1193             *outbuf = 0;
1194         }
1195         else
1196         {
1197             sizez = term->u.general->len;
1198             if (sizez > IT_MAX_WORD-1)
1199                 sizez = IT_MAX_WORD-1;
1200             memcpy (termz, term->u.general->buf, sizez);
1201             termz[sizez] = '\0';
1202         }
1203         break;
1204     case Z_Term_characterString:
1205         sizez = strlen(term->u.characterString);
1206         if (sizez > IT_MAX_WORD-1)
1207             sizez = IT_MAX_WORD-1;
1208         memcpy (termz, term->u.characterString, sizez);
1209         termz[sizez] = '\0';
1210         break;
1211     default:
1212         zh->errCode = 124;
1213         return -1;
1214     }
1215     return 0;
1216 }
1217
1218 /* convert APT SCAN term to internal cmap */
1219 static int trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1220                             char *termz, int reg_type)
1221 {
1222     char termz0[IT_MAX_WORD];
1223
1224     if (zapt_term_to_utf8(zh, zapt, termz0))
1225         return -1;    /* error */
1226     else
1227     {
1228         const char **map;
1229         const char *cp = (const char *) termz0;
1230         const char *cp_end = cp + strlen(cp);
1231         const char *src;
1232         int i = 0;
1233         const char *space_map = NULL;
1234         int len;
1235
1236         while ((len = (cp_end - cp)) > 0)
1237         {
1238             map = zebra_maps_input (zh->reg->zebra_maps, reg_type, &cp, len, 0);
1239             if (**map == *CHR_SPACE)
1240                 space_map = *map;
1241             else
1242             {
1243                 if (i && space_map)
1244                     for (src = space_map; *src; src++)
1245                         termz[i++] = *src;
1246                 space_map = NULL;
1247                 for (src = *map; *src; src++)
1248                     termz[i++] = *src;
1249             }
1250         }
1251         termz[i] = '\0';
1252     }
1253     return 0;
1254 }
1255
1256 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1257                      const char *termz, NMEM stream, unsigned reg_id)
1258 {
1259     WRBUF wrbuf = 0;
1260     AttrType truncation;
1261     int truncation_value;
1262     char *ex_list = 0;
1263
1264     attr_init (&truncation, zapt, 5);
1265     truncation_value = attr_find (&truncation, NULL);
1266
1267     switch (truncation_value)
1268     {
1269     default:
1270         ex_list = "";
1271         break;
1272     case 101:
1273         ex_list = "#";
1274         break;
1275     case 102:
1276     case 103:
1277         ex_list = 0;
1278         break;
1279     case 104:
1280         ex_list = "!#";
1281         break;
1282     case 105:
1283         ex_list = "!*";
1284         break;
1285     }
1286     if (ex_list)
1287         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1288                               termz, strlen(termz));
1289     if (!wrbuf)
1290         return nmem_strdup(stream, termz);
1291     else
1292     {
1293         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1294         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1295         buf[wrbuf_len(wrbuf)] = '\0';
1296         return buf;
1297     }
1298 }
1299
1300 static void grep_info_delete (struct grep_info *grep_info)
1301 {
1302 #ifdef TERM_COUNT
1303     xfree(grep_info->term_no);
1304 #endif
1305     xfree (grep_info->isam_p_buf);
1306 }
1307
1308 static int grep_info_prepare (ZebraHandle zh,
1309                               Z_AttributesPlusTerm *zapt,
1310                               struct grep_info *grep_info,
1311                               int reg_type,
1312                               NMEM stream)
1313 {
1314     AttrType termset;
1315     int termset_value_numeric;
1316     const char *termset_value_string;
1317
1318 #ifdef TERM_COUNT
1319     grep_info->term_no = 0;
1320 #endif
1321     grep_info->isam_p_size = 0;
1322     grep_info->isam_p_buf = NULL;
1323     grep_info->zh = zh;
1324     grep_info->reg_type = reg_type;
1325     grep_info->termset = 0;
1326
1327     if (!zapt)
1328         return 0;
1329     attr_init (&termset, zapt, 8);
1330     termset_value_numeric =
1331         attr_find_ex (&termset, NULL, &termset_value_string);
1332     if (termset_value_numeric != -1)
1333     {
1334         char resname[32];
1335         const char *termset_name = 0;
1336         if (termset_value_numeric != -2)
1337         {
1338
1339             sprintf (resname, "%d", termset_value_numeric);
1340             termset_name = resname;
1341         }
1342         else
1343             termset_name = termset_value_string;
1344         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1345         grep_info->termset = resultSetAdd (zh, termset_name, 1);
1346         if (!grep_info->termset)
1347         {
1348             zh->errCode = 128;
1349             zh->errString = nmem_strdup (stream, termset_name);
1350             return -1;
1351         }
1352     }
1353     return 0;
1354 }
1355
1356
1357 static RSET rpn_search_APT_phrase (ZebraHandle zh,
1358                                    Z_AttributesPlusTerm *zapt,
1359                                    const char *termz_org,
1360                                    oid_value attributeSet,
1361                                    NMEM stream,
1362                                    int reg_type, int complete_flag,
1363                                    const char *rank_type, int xpath_use,
1364                                    int num_bases, char **basenames,
1365                                    NMEM rset_nmem)
1366 {
1367     char term_dst[IT_MAX_WORD+1];
1368     RSET rset[60], result;
1369     int rset_no = 0;
1370     struct grep_info grep_info;
1371     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1372     const char *termp = termz;
1373
1374     *term_dst = 0;
1375     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1376         return 0;
1377     while (1)
1378     {
1379         yaz_log(log_level_rpn, "APT_phrase termp=%s", termp);
1380         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1381                                     stream, &grep_info,
1382                                     reg_type, complete_flag,
1383                                     num_bases, basenames,
1384                                     term_dst, rank_type,
1385                                     xpath_use,rset_nmem);
1386         if (!rset[rset_no])
1387             break;
1388         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1389             break;
1390     }
1391     grep_info_delete (&grep_info);
1392     if (rset_no == 0)
1393         return rsnull_create (rset_nmem,key_it_ctrl);
1394     else if (rset_no == 1)
1395         return (rset[0]);
1396     else
1397         result = rsprox_create( rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1398                        rset_no, rset,
1399                        1 /* ordered */, 0 /* exclusion */,
1400                        3 /* relation */, 1 /* distance */);
1401     return result;
1402 }
1403
1404 static RSET rpn_search_APT_or_list (ZebraHandle zh,
1405                                     Z_AttributesPlusTerm *zapt,
1406                                     const char *termz_org,
1407                                     oid_value attributeSet,
1408                                     NMEM stream,
1409                                     int reg_type, int complete_flag,
1410                                     const char *rank_type,
1411                                     int xpath_use,
1412                                     int num_bases, char **basenames,
1413                                     NMEM rset_nmem)
1414 {
1415     char term_dst[IT_MAX_WORD+1];
1416     RSET rset[60];
1417     int rset_no = 0;
1418     struct grep_info grep_info;
1419     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1420     const char *termp = termz;
1421
1422     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1423         return 0;
1424     while (1)
1425     {
1426         yaz_log(log_level_rpn, "APT_or_list termp=%s", termp);
1427         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1428                                     stream, &grep_info,
1429                                     reg_type, complete_flag,
1430                                     num_bases, basenames,
1431                                     term_dst, rank_type,
1432                                     xpath_use,rset_nmem);
1433         if (!rset[rset_no])
1434             break;
1435         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1436             break;
1437     }
1438     grep_info_delete (&grep_info);
1439     if (rset_no == 0)
1440         return rsnull_create (rset_nmem,key_it_ctrl);
1441     return rsmultior_create(rset_nmem, key_it_ctrl,key_it_ctrl->scope,
1442                             rset_no, rset);
1443 }
1444
1445 static RSET rpn_search_APT_and_list (ZebraHandle zh,
1446                                      Z_AttributesPlusTerm *zapt,
1447                                      const char *termz_org,
1448                                      oid_value attributeSet,
1449                                      NMEM stream,
1450                                      int reg_type, int complete_flag,
1451                                      const char *rank_type,
1452                                      int xpath_use,
1453                                      int num_bases, char **basenames,
1454                                      NMEM rset_nmem)
1455 {
1456     char term_dst[IT_MAX_WORD+1];
1457     RSET rset[60]; /* FIXME - bug 160 - should be dynamic somehow */
1458     int rset_no = 0;
1459     struct grep_info grep_info;
1460     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1461     const char *termp = termz;
1462
1463     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1464         return 0;
1465     while (1)
1466     {
1467         yaz_log(log_level_rpn, "APT_and_list termp=%s", termp);
1468         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1469                                     stream, &grep_info,
1470                                     reg_type, complete_flag,
1471                                     num_bases, basenames,
1472                                     term_dst, rank_type,
1473                                     xpath_use, rset_nmem);
1474         if (!rset[rset_no])
1475             break;
1476         assert (rset[rset_no]);
1477         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1478             break;
1479     }
1480     grep_info_delete (&grep_info);
1481     if (rset_no == 0)
1482         return rsnull_create (rset_nmem,key_it_ctrl);
1483
1484     return rsmultiand_create( rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1485                               rset_no, rset);
1486 }
1487
1488 static int numeric_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1489                              const char **term_sub,
1490                              char *term_dict,
1491                              oid_value attributeSet,
1492                              struct grep_info *grep_info,
1493                              int *max_pos,
1494                              int reg_type,
1495                              char *term_dst)
1496 {
1497     AttrType relation;
1498     int relation_value;
1499     int term_value;
1500     int r;
1501     char *term_tmp = term_dict + strlen(term_dict);
1502
1503     attr_init (&relation, zapt, 2);
1504     relation_value = attr_find (&relation, NULL);
1505
1506     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1507
1508     if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1509                    term_dst))
1510         return 0;
1511     term_value = atoi (term_tmp);
1512     switch (relation_value)
1513     {
1514     case 1:
1515         yaz_log(log_level_rpn, "Relation <");
1516         gen_regular_rel (term_tmp, term_value-1, 1);
1517         break;
1518     case 2:
1519         yaz_log(log_level_rpn, "Relation <=");
1520         gen_regular_rel (term_tmp, term_value, 1);
1521         break;
1522     case 4:
1523         yaz_log(log_level_rpn, "Relation >=");
1524         gen_regular_rel (term_tmp, term_value, 0);
1525         break;
1526     case 5:
1527         yaz_log(log_level_rpn, "Relation >");
1528         gen_regular_rel (term_tmp, term_value+1, 0);
1529         break;
1530     case 3:
1531     default:
1532         yaz_log(log_level_rpn, "Relation =");
1533         sprintf (term_tmp, "(0*%d)", term_value);
1534     }
1535     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1536     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1537                           0, grep_handle);
1538     if (r)
1539         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
1540     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1541     return 1;
1542 }
1543
1544 static int numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1545                          const char **term_sub,
1546                          oid_value attributeSet, struct grep_info *grep_info,
1547                          int reg_type, int complete_flag,
1548                          int num_bases, char **basenames,
1549                          char *term_dst, int xpath_use, NMEM stream)
1550 {
1551     char term_dict[2*IT_MAX_WORD+2];
1552     int r, base_no;
1553     AttrType use;
1554     int use_value;
1555     const char *use_string = 0;
1556     oid_value curAttributeSet = attributeSet;
1557     const char *termp;
1558     struct rpn_char_map_info rcmi;
1559
1560     int bases_ok = 0;     /* no of databases with OK attribute */
1561     int errCode = 0;      /* err code (if any is not OK) */
1562     char *errString = 0;  /* addinfo */
1563
1564     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1565     attr_init (&use, zapt, 1);
1566     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1567
1568     if (use_value == -1)
1569         use_value = 1016;
1570
1571     for (base_no = 0; base_no < num_bases; base_no++)
1572     {
1573         attent attp;
1574         data1_local_attribute id_xpath_attr;
1575         data1_local_attribute *local_attr;
1576         int max_pos, prefix_len = 0;
1577
1578         termp = *term_sub;
1579         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1580         {
1581             use_value = xpath_use;
1582             attp.local_attributes = &id_xpath_attr;
1583             attp.attset_ordinal = VAL_IDXPATH;
1584             id_xpath_attr.next = 0;
1585             id_xpath_attr.local = use_value;
1586         }
1587         else if (curAttributeSet == VAL_IDXPATH)
1588         {
1589             attp.local_attributes = &id_xpath_attr;
1590             attp.attset_ordinal = VAL_IDXPATH;
1591             id_xpath_attr.next = 0;
1592             id_xpath_attr.local = use_value;
1593         }
1594         else
1595         {
1596             if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1597                                             use_string)))
1598             {
1599                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1600                       curAttributeSet, use_value, r);
1601                 if (r == -1)
1602                 {
1603                     errString = nmem_strdup_i(stream, use_value);
1604                     errCode = 114;
1605                 }
1606                 else
1607                     errCode = 121;
1608                 continue;
1609             }
1610         }
1611         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1612         {
1613             zh->errCode = 109; /* Database unavailable */
1614             zh->errString = basenames[base_no];
1615             return -1;
1616         }
1617         for (local_attr = attp.local_attributes; local_attr;
1618              local_attr = local_attr->next)
1619         {
1620             int ord;
1621             char ord_buf[32];
1622             int i, ord_len;
1623
1624             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
1625                                           local_attr->local);
1626             if (ord < 0)
1627                 continue;
1628             if (prefix_len)
1629                 term_dict[prefix_len++] = '|';
1630             else
1631                 term_dict[prefix_len++] = '(';
1632
1633             ord_len = key_SU_encode (ord, ord_buf);
1634             for (i = 0; i<ord_len; i++)
1635             {
1636                 term_dict[prefix_len++] = 1;
1637                 term_dict[prefix_len++] = ord_buf[i];
1638             }
1639         }
1640         if (!prefix_len)
1641         {
1642             errCode = 114;
1643             errString = nmem_strdup_i(stream, use_value);
1644             continue;
1645         }
1646         bases_ok++;
1647         term_dict[prefix_len++] = ')';
1648         term_dict[prefix_len++] = 1;
1649         term_dict[prefix_len++] = reg_type;
1650         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1651         term_dict[prefix_len] = '\0';
1652         if (!numeric_relation (zh, zapt, &termp, term_dict,
1653                                attributeSet, grep_info, &max_pos, reg_type,
1654                                term_dst))
1655             return 0;
1656     }
1657     if (!bases_ok)
1658     {
1659         zh->errCode = errCode;
1660         zh->errString = errString;
1661         return -1;
1662     }
1663     *term_sub = termp;
1664     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1665     return 1;
1666 }
1667
1668 static RSET rpn_search_APT_numeric (ZebraHandle zh,
1669                                     Z_AttributesPlusTerm *zapt,
1670                                     const char *termz,
1671                                     oid_value attributeSet,
1672                                     NMEM stream,
1673                                     int reg_type, int complete_flag,
1674                                     const char *rank_type, int xpath_use,
1675                                     int num_bases, char **basenames,
1676                                     NMEM rset_nmem)
1677 {
1678     char term_dst[IT_MAX_WORD+1];
1679     const char *termp = termz;
1680     RSET rset[60]; /* FIXME - hard-coded magic number */
1681     int  r, rset_no = 0;
1682     struct grep_info grep_info;
1683
1684     yaz_log(log_level_rpn, "APT_numeric t='%s'",termz);
1685     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1686         return 0;
1687     while (1)
1688     {
1689         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1690         grep_info.isam_p_indx = 0;
1691         r = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1692                           reg_type, complete_flag, num_bases, basenames,
1693                           term_dst, xpath_use,
1694                           stream);
1695         if (r < 1)
1696             break;
1697         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1698         rset[rset_no] = rset_trunc(zh, grep_info.isam_p_buf,
1699                                     grep_info.isam_p_indx, term_dst,
1700                                     strlen(term_dst), rank_type,
1701                                     0 /* preserve position */,
1702                                     zapt->term->which, rset_nmem,
1703                                     key_it_ctrl,key_it_ctrl->scope);
1704         assert (rset[rset_no]);
1705         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1706             break;
1707     }
1708     grep_info_delete (&grep_info);
1709     if (rset_no == 0)
1710         return rsnull_create (rset_nmem,key_it_ctrl);
1711     if (rset_no == 1)
1712         return rset[0];
1713     return rsmultiand_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1714                rset_no, rset);
1715 }
1716
1717 static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1718                                   const char *termz,
1719                                   oid_value attributeSet,
1720                                   NMEM stream,
1721                                   const char *rank_type, NMEM rset_nmem)
1722 {
1723     RSET result;
1724     RSFD rsfd;
1725     struct it_key key;
1726     int sys;
1727     result = rstemp_create( rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1728                      res_get (zh->res, "setTmpDir"),0 );
1729     rsfd = rset_open (result, RSETF_WRITE);
1730
1731     sys = atoi(termz);
1732     if (sys <= 0)
1733         sys = 1;
1734     key.mem[0] = sys;
1735     key.mem[1] = 1;
1736     key.len = 2;
1737     rset_write (rsfd, &key);
1738     rset_close (rsfd);
1739     return result;
1740 }
1741
1742 static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1743                            oid_value attributeSet, NMEM stream,
1744                            Z_SortKeySpecList *sort_sequence,
1745                            const char *rank_type)
1746 {
1747     int i;
1748     int sort_relation_value;
1749     AttrType sort_relation_type;
1750     int use_value;
1751     AttrType use_type;
1752     Z_SortKeySpec *sks;
1753     Z_SortKey *sk;
1754     Z_AttributeElement *ae;
1755     int oid[OID_SIZE];
1756     oident oe;
1757     char termz[20];
1758
1759     attr_init (&sort_relation_type, zapt, 7);
1760     sort_relation_value = attr_find (&sort_relation_type, &attributeSet);
1761
1762     attr_init (&use_type, zapt, 1);
1763     use_value = attr_find (&use_type, &attributeSet);
1764
1765     if (!sort_sequence->specs)
1766     {
1767         sort_sequence->num_specs = 10;
1768         sort_sequence->specs = (Z_SortKeySpec **)
1769             nmem_malloc(stream, sort_sequence->num_specs *
1770                          sizeof(*sort_sequence->specs));
1771         for (i = 0; i<sort_sequence->num_specs; i++)
1772             sort_sequence->specs[i] = 0;
1773     }
1774     if (zapt->term->which != Z_Term_general)
1775         i = 0;
1776     else
1777         i = atoi_n ((char *) zapt->term->u.general->buf,
1778                     zapt->term->u.general->len);
1779     if (i >= sort_sequence->num_specs)
1780         i = 0;
1781     sprintf (termz, "%d", i);
1782
1783     oe.proto = PROTO_Z3950;
1784     oe.oclass = CLASS_ATTSET;
1785     oe.value = attributeSet;
1786     if (!oid_ent_to_oid (&oe, oid))
1787         return 0;
1788
1789     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1790     sks->sortElement = (Z_SortElement *)
1791         nmem_malloc(stream, sizeof(*sks->sortElement));
1792     sks->sortElement->which = Z_SortElement_generic;
1793     sk = sks->sortElement->u.generic = (Z_SortKey *)
1794         nmem_malloc(stream, sizeof(*sk));
1795     sk->which = Z_SortKey_sortAttributes;
1796     sk->u.sortAttributes = (Z_SortAttributes *)
1797         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1798
1799     sk->u.sortAttributes->id = oid;
1800     sk->u.sortAttributes->list = (Z_AttributeList *)
1801         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
1802     sk->u.sortAttributes->list->num_attributes = 1;
1803     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
1804         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
1805     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
1806         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
1807     ae->attributeSet = 0;
1808     ae->attributeType = (int *)
1809         nmem_malloc(stream, sizeof(*ae->attributeType));
1810     *ae->attributeType = 1;
1811     ae->which = Z_AttributeValue_numeric;
1812     ae->value.numeric = (int *)
1813         nmem_malloc(stream, sizeof(*ae->value.numeric));
1814     *ae->value.numeric = use_value;
1815
1816     sks->sortRelation = (int *)
1817         nmem_malloc(stream, sizeof(*sks->sortRelation));
1818     if (sort_relation_value == 1)
1819         *sks->sortRelation = Z_SortKeySpec_ascending;
1820     else if (sort_relation_value == 2)
1821         *sks->sortRelation = Z_SortKeySpec_descending;
1822     else
1823         *sks->sortRelation = Z_SortKeySpec_ascending;
1824
1825     sks->caseSensitivity = (int *)
1826         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1827     *sks->caseSensitivity = 0;
1828
1829     sks->which = Z_SortKeySpec_null;
1830     sks->u.null = odr_nullval ();
1831     sort_sequence->specs[i] = sks;
1832     return rsnull_create (NULL,key_it_ctrl);
1833         /* FIXME - nmem?? */
1834 }
1835
1836
1837 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1838                        oid_value attributeSet,
1839                        struct xpath_location_step *xpath, int max, NMEM mem)
1840 {
1841     oid_value curAttributeSet = attributeSet;
1842     AttrType use;
1843     const char *use_string = 0;
1844
1845     attr_init (&use, zapt, 1);
1846     attr_find_ex (&use, &curAttributeSet, &use_string);
1847
1848     if (!use_string || *use_string != '/')
1849         return -1;
1850
1851     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1852 }
1853
1854
1855
1856 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1857                         int reg_type, const char *term, int use,
1858                         oid_value curAttributeSet, NMEM rset_nmem)
1859 {
1860     RSET rset;
1861     struct grep_info grep_info;
1862     char term_dict[2048];
1863     char ord_buf[32];
1864     int prefix_len = 0;
1865     int ord = zebraExplain_lookupSU (zh->reg->zei, curAttributeSet, use);
1866     int ord_len, i, r, max_pos;
1867     int term_type = Z_Term_characterString;
1868     const char *flags = "void";
1869
1870     if (grep_info_prepare (zh, 0 /* zapt */, &grep_info, '0', stream))
1871         return rsnull_create (rset_nmem,key_it_ctrl);
1872
1873     if (ord < 0)
1874         return rsnull_create (rset_nmem,key_it_ctrl);
1875     if (prefix_len)
1876         term_dict[prefix_len++] = '|';
1877     else
1878         term_dict[prefix_len++] = '(';
1879
1880     ord_len = key_SU_encode (ord, ord_buf);
1881     for (i = 0; i<ord_len; i++)
1882     {
1883         term_dict[prefix_len++] = 1;
1884         term_dict[prefix_len++] = ord_buf[i];
1885     }
1886     term_dict[prefix_len++] = ')';
1887     term_dict[prefix_len++] = 1;
1888     term_dict[prefix_len++] = reg_type;
1889
1890     strcpy(term_dict+prefix_len, term);
1891
1892     grep_info.isam_p_indx = 0;
1893     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
1894                           &grep_info, &max_pos, 0, grep_handle);
1895     yaz_log (YLOG_LOG, "%s %d positions", term,
1896              grep_info.isam_p_indx);
1897     rset = rset_trunc(zh, grep_info.isam_p_buf,
1898                        grep_info.isam_p_indx, term, strlen(term),
1899                        flags, 1, term_type,rset_nmem,
1900                        key_it_ctrl, key_it_ctrl->scope);
1901     grep_info_delete (&grep_info);
1902     return rset;
1903 }
1904
1905 static RSET rpn_search_xpath (ZebraHandle zh,
1906                               oid_value attributeSet,
1907                               int num_bases, char **basenames,
1908                               NMEM stream, const char *rank_type, RSET rset,
1909                               int xpath_len, struct xpath_location_step *xpath,
1910                               NMEM rset_nmem)
1911 {
1912     oid_value curAttributeSet = attributeSet;
1913     int base_no;
1914     int i;
1915
1916     if (xpath_len < 0)
1917         return rset;
1918
1919     yaz_log (YLOG_DEBUG, "xpath len=%d", xpath_len);
1920     for (i = 0; i<xpath_len; i++)
1921     {
1922         yaz_log (log_level_rpn, "XPATH %d %s", i, xpath[i].part);
1923
1924     }
1925
1926     curAttributeSet = VAL_IDXPATH;
1927
1928     /*
1929       //a    ->    a/.*
1930       //a/b  ->    b/a/.*
1931       /a     ->    a/
1932       /a/b   ->    b/a/
1933
1934       /      ->    none
1935
1936    a[@attr=value]/b[@other=othervalue]
1937
1938  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
1939  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
1940  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
1941  /a/b[@c=y] val range(b/a/,freetext(w,1016,val),b/a/,@c=y)
1942  /a[@c=y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c=y)
1943  /a[@c=x]/b[@c=y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c=y),a/,@c=x)
1944
1945     */
1946
1947     dict_grep_cmap (zh->reg->dict, 0, 0);
1948
1949     for (base_no = 0; base_no < num_bases; base_no++)
1950     {
1951         int level = xpath_len;
1952         int first_path = 1;
1953
1954         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1955         {
1956             zh->errCode = 109; /* Database unavailable */
1957             zh->errString = basenames[base_no];
1958             return rset;
1959         }
1960         while (--level >= 0)
1961         {
1962             char xpath_rev[128];
1963             int i, len;
1964             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
1965
1966             *xpath_rev = 0;
1967             len = 0;
1968             for (i = level; i >= 1; --i)
1969             {
1970                 const char *cp = xpath[i].part;
1971                 if (*cp)
1972                 {
1973                     for (;*cp; cp++)
1974                         if (*cp == '*')
1975                         {
1976                             memcpy (xpath_rev + len, "[^/]*", 5);
1977                             len += 5;
1978                         }
1979                         else if (*cp == ' ')
1980                         {
1981
1982                             xpath_rev[len++] = 1;
1983                             xpath_rev[len++] = ' ';
1984                         }
1985
1986                         else
1987                             xpath_rev[len++] = *cp;
1988                     xpath_rev[len++] = '/';
1989                 }
1990                 else if (i == 1)  /* // case */
1991                 {
1992                     xpath_rev[len++] = '.';
1993                     xpath_rev[len++] = '*';
1994                 }
1995             }
1996             xpath_rev[len] = 0;
1997
1998             if (xpath[level].predicate &&
1999                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2000                 xpath[level].predicate->u.relation.name[0])
2001             {
2002                 WRBUF wbuf = wrbuf_alloc();
2003                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2004                 if (xpath[level].predicate->u.relation.value)
2005                 {
2006                     const char *cp = xpath[level].predicate->u.relation.value;
2007                     wrbuf_putc(wbuf, '=');
2008
2009                     while (*cp)
2010                     {
2011                         if (strchr(REGEX_CHARS, *cp))
2012                             wrbuf_putc(wbuf, '\\');
2013                         wrbuf_putc(wbuf, *cp);
2014                         cp++;
2015                     }
2016                 }
2017                 wrbuf_puts(wbuf, "");
2018                 rset_attr = xpath_trunc(
2019                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2020                     curAttributeSet,rset_nmem);
2021                 wrbuf_free(wbuf, 1);
2022             }
2023             else
2024             {
2025                 if (!first_path)
2026                     continue;
2027             }
2028             yaz_log (log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2029             if (strlen(xpath_rev))
2030             {
2031                 rset_start_tag = xpath_trunc(zh, stream, '0',
2032                         xpath_rev, 1, curAttributeSet, rset_nmem);
2033
2034                 rset_end_tag = xpath_trunc(zh, stream, '0',
2035                         xpath_rev, 2, curAttributeSet, rset_nmem);
2036
2037                 /*
2038                 parms.key_size = sizeof(struct it_key);
2039                 parms.cmp = key_compare_it;
2040                 parms.rset_l = rset_start_tag;
2041                 parms.rset_m = rset;
2042                 parms.rset_r = rset_end_tag;
2043                 parms.rset_attr = rset_attr;
2044                 parms.printer = key_print_it;
2045                 rset = rset_create (rset_kind_between, &parms);
2046                 */
2047                 rset=rsbetween_create( rset_nmem,key_it_ctrl,key_it_ctrl->scope,
2048                         rset_start_tag, rset, rset_end_tag, rset_attr);
2049             }
2050             first_path = 0;
2051         }
2052     }
2053
2054     return rset;
2055 }
2056
2057
2058
2059 static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2060                             oid_value attributeSet, NMEM stream,
2061                             Z_SortKeySpecList *sort_sequence,
2062                             int num_bases, char **basenames,
2063                             NMEM rset_nmem)
2064 {
2065     unsigned reg_id;
2066     char *search_type = NULL;
2067     char rank_type[128];
2068     int complete_flag;
2069     int sort_flag;
2070     char termz[IT_MAX_WORD+1];
2071     RSET rset = 0;
2072     int xpath_len;
2073     int xpath_use = 0;
2074     struct xpath_location_step xpath[10];
2075
2076     if (!log_level_set)
2077     {
2078         log_level_rpn = yaz_log_module_level("rpn");
2079         log_level_set=1;
2080     }
2081     zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2082                      rank_type, &complete_flag, &sort_flag);
2083
2084     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2085     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2086     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2087     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2088
2089     if (zapt_term_to_utf8(zh, zapt, termz))
2090         return 0;
2091
2092     if (sort_flag)
2093         return rpn_sort_spec (zh, zapt, attributeSet, stream, sort_sequence,
2094                               rank_type);
2095     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2096     if (xpath_len >= 0)
2097     {
2098         xpath_use = 1016;
2099         if (xpath[xpath_len-1].part[0] == '@')
2100             xpath_use = 1015;
2101     }
2102
2103     if (!strcmp (search_type, "phrase"))
2104     {
2105         rset = rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
2106                                       reg_id, complete_flag, rank_type,
2107                                       xpath_use,
2108                                       num_bases, basenames, rset_nmem);
2109     }
2110     else if (!strcmp (search_type, "and-list"))
2111     {
2112         rset = rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
2113                                         reg_id, complete_flag, rank_type,
2114                                         xpath_use,
2115                                         num_bases, basenames, rset_nmem);
2116     }
2117     else if (!strcmp (search_type, "or-list"))
2118     {
2119         rset = rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
2120                                        reg_id, complete_flag, rank_type,
2121                                        xpath_use,
2122                                        num_bases, basenames, rset_nmem);
2123     }
2124     else if (!strcmp (search_type, "local"))
2125     {
2126         rset = rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
2127                                      rank_type, rset_nmem);
2128     }
2129     else if (!strcmp (search_type, "numeric"))
2130     {
2131         rset = rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
2132                                        reg_id, complete_flag, rank_type,
2133                                        xpath_use,
2134                                        num_bases, basenames, rset_nmem);
2135     }
2136     else if (!strcmp (search_type, "always"))
2137     {
2138         rset = 0;
2139     }
2140     else
2141         zh->errCode = 118;
2142     return rpn_search_xpath (zh, attributeSet, num_bases, basenames,
2143                              stream, rank_type, rset,
2144                              xpath_len, xpath, rset_nmem);
2145 }
2146
2147 static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
2148                                   oid_value attributeSet,
2149                                   NMEM stream, NMEM rset_nmem,
2150                                   Z_SortKeySpecList *sort_sequence,
2151                                   int num_bases, char **basenames)
2152 {
2153     RSET r = NULL;
2154     if (zs->which == Z_RPNStructure_complex)
2155     {
2156         Z_Operator *zop = zs->u.complex->roperator;
2157         RSET rsets[2]; /* l and r argument */
2158
2159         rsets[0]=rpn_search_structure (zh, zs->u.complex->s1,
2160                                        attributeSet, stream, rset_nmem,
2161                                        sort_sequence,
2162                                        num_bases, basenames);
2163         if (rsets[0] == NULL)
2164             return NULL;
2165         rsets[1]=rpn_search_structure (zh, zs->u.complex->s2,
2166                                        attributeSet, stream, rset_nmem,
2167                                        sort_sequence,
2168                                        num_bases, basenames);
2169         if (rsets[1] == NULL)
2170         {
2171             rset_delete (rsets[0]);
2172             return NULL;
2173         }
2174
2175         switch (zop->which)
2176         {
2177         case Z_Operator_and:
2178             r=rsmultiand_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2179                         2, rsets);
2180             break;
2181         case Z_Operator_or:
2182             r=rsmultior_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2183                         2, rsets);
2184             break;
2185         case Z_Operator_and_not:
2186             r = rsbool_create_not(rset_nmem,key_it_ctrl, key_it_ctrl->scope,
2187                     rsets[0],rsets[1]);
2188             break;
2189         case Z_Operator_prox:
2190             if (zop->u.prox->which != Z_ProximityOperator_known)
2191             {
2192                 zh->errCode = 132;
2193                 return NULL;
2194             }
2195             if (*zop->u.prox->u.known != Z_ProxUnit_word)
2196             {
2197                 char *val = (char *) nmem_malloc(stream, 16);
2198                 zh->errCode = 132;
2199                 zh->errString = val;
2200                 sprintf (val, "%d", *zop->u.prox->u.known);
2201                 return NULL;
2202             }
2203             else
2204             {
2205                 /* new / old prox */
2206                 r=rsprox_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
2207                          2, rsets,
2208                          *zop->u.prox->ordered,
2209                          (!zop->u.prox->exclusion ?
2210                               0 : *zop->u.prox->exclusion),
2211                          *zop->u.prox->relationType,
2212                          *zop->u.prox->distance );
2213             }
2214             break;
2215         default:
2216             zh->errCode = 110;
2217             return NULL;
2218         }
2219     }
2220     else if (zs->which == Z_RPNStructure_simple)
2221     {
2222         if (zs->u.simple->which == Z_Operand_APT)
2223         {
2224             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2225             r = rpn_search_APT (zh, zs->u.simple->u.attributesPlusTerm,
2226                                 attributeSet, stream, sort_sequence,
2227                                 num_bases, basenames,rset_nmem);
2228         }
2229         else if (zs->u.simple->which == Z_Operand_resultSetId)
2230         {
2231             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2232             r = resultSetRef (zh, zs->u.simple->u.resultSetId);
2233             if (!r)
2234             {
2235                 r = rsnull_create (rset_nmem,key_it_ctrl);
2236                 zh->errCode = 30;
2237                 zh->errString =
2238                     nmem_strdup (stream, zs->u.simple->u.resultSetId);
2239                 return 0;
2240             }
2241             else
2242                 rset_dup(r);
2243         }
2244         else
2245         {
2246             zh->errCode = 3;
2247             return 0;
2248         }
2249     }
2250     else
2251     {
2252         zh->errCode = 3;
2253         return 0;
2254     }
2255     return r;
2256 }
2257
2258
2259 RSET rpn_search(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
2260                 Z_RPNQuery *rpn, int num_bases, char **basenames,
2261                 const char *setname,
2262                 ZebraSet sset)
2263 {
2264     RSET rset;
2265     oident *attrset;
2266     oid_value attributeSet;
2267     Z_SortKeySpecList *sort_sequence;
2268     int sort_status, i;
2269
2270     zh->errCode = 0;
2271     zh->errString = NULL;
2272     zh->hits = 0;
2273
2274     sort_sequence = (Z_SortKeySpecList *)
2275         nmem_malloc(nmem, sizeof(*sort_sequence));
2276     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
2277     sort_sequence->specs = (Z_SortKeySpec **)
2278         nmem_malloc(nmem, sort_sequence->num_specs *
2279                      sizeof(*sort_sequence->specs));
2280     for (i = 0; i<sort_sequence->num_specs; i++)
2281         sort_sequence->specs[i] = 0;
2282
2283     attrset = oid_getentbyoid (rpn->attributeSetId);
2284     attributeSet = attrset->value;
2285     rset = rpn_search_structure (zh, rpn->RPNStructure, attributeSet,
2286                                  nmem, rset_nmem,
2287                                  sort_sequence, num_bases, basenames);
2288     if (!rset)
2289         return 0;
2290
2291     if (zh->errCode)
2292         yaz_log(YLOG_DEBUG, "search error: %d", zh->errCode);
2293
2294     for (i = 0; sort_sequence->specs[i]; i++)
2295         ;
2296     sort_sequence->num_specs = i;
2297     if (!i)
2298         resultSetRank (zh, sset, rset, rset_nmem);
2299     else
2300     {
2301         yaz_log(YLOG_DEBUG, "resultSetSortSingle in rpn_search");
2302         resultSetSortSingle (zh, nmem, sset, rset,
2303                              sort_sequence, &sort_status);
2304         if (zh->errCode)
2305         {
2306             yaz_log(YLOG_DEBUG, "resultSetSortSingle status = %d", zh->errCode);
2307         }
2308     }
2309     return rset;
2310 }
2311
2312 struct scan_info_entry {
2313     char *term;
2314     ISAMC_P isam_p;
2315 };
2316
2317 struct scan_info {
2318     struct scan_info_entry *list;
2319     ODR odr;
2320     int before, after;
2321     char prefix[20];
2322 };
2323
2324 static int scan_handle (char *name, const char *info, int pos, void *client)
2325 {
2326     int len_prefix, idx;
2327     struct scan_info *scan_info = (struct scan_info *) client;
2328
2329     len_prefix = strlen(scan_info->prefix);
2330     if (memcmp (name, scan_info->prefix, len_prefix))
2331         return 1;
2332     if (pos > 0)        idx = scan_info->after - pos + scan_info->before;
2333     else
2334         idx = - pos - 1;
2335     scan_info->list[idx].term = (char *)
2336         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2337     strcpy(scan_info->list[idx].term, name + len_prefix);
2338     assert (*info == sizeof(ISAMC_P));
2339     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAMC_P));
2340     return 0;
2341 }
2342
2343 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2344                                char **dst, const char *src)
2345 {
2346     char term_src[IT_MAX_WORD];
2347     char term_dst[IT_MAX_WORD];
2348
2349     term_untrans (zh, reg_type, term_src, src);
2350
2351     if (zh->iconv_from_utf8 != 0)
2352     {
2353         int len;
2354         char *inbuf = term_src;
2355         size_t inleft = strlen(term_src);
2356         char *outbuf = term_dst;
2357         size_t outleft = sizeof(term_dst)-1;
2358         size_t ret;
2359
2360         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2361                          &outbuf, &outleft);
2362         if (ret == (size_t)(-1))
2363             len = 0;
2364         else
2365             len = outbuf - term_dst;
2366         *dst = nmem_malloc(stream, len + 1);
2367         if (len > 0)
2368             memcpy (*dst, term_dst, len);
2369         (*dst)[len] = '\0';
2370     }
2371     else
2372         *dst = nmem_strdup(stream, term_src);
2373 }
2374
2375 static void count_set (RSET r, int *count)
2376 {
2377     zint psysno = 0;
2378     int kno = 0;
2379     struct it_key key;
2380     RSFD rfd;
2381
2382     yaz_log(YLOG_DEBUG, "count_set");
2383
2384     *count = 0;
2385     rfd = rset_open (r, RSETF_READ);
2386     while (rset_read (rfd, &key,0 /* never mind terms */))
2387     {
2388         if (key.mem[0] != psysno)
2389         {
2390             psysno = key.mem[0];
2391             (*count)++;
2392         }
2393         kno++;
2394     }
2395     rset_close (rfd);
2396     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2397 }
2398
2399 void rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2400                oid_value attributeset,
2401                int num_bases, char **basenames,
2402                int *position, int *num_entries, ZebraScanEntry **list,
2403                int *is_partial, RSET limit_set, int return_zero)
2404 {
2405     int i;
2406     int pos = *position;
2407     int num = *num_entries;
2408     int before;
2409     int after;
2410     int base_no;
2411     char termz[IT_MAX_WORD+20];
2412     AttrType use;
2413     int use_value;
2414     const char *use_string = 0;
2415     struct scan_info *scan_info_array;
2416     ZebraScanEntry *glist;
2417     int ords[32], ord_no = 0;
2418     int ptr[32];
2419
2420     int bases_ok = 0;     /* no of databases with OK attribute */
2421     int errCode = 0;      /* err code (if any is not OK) */
2422     char *errString = 0;  /* addinfo */
2423
2424     unsigned reg_id;
2425     char *search_type = NULL;
2426     char rank_type[128];
2427     int complete_flag;
2428     int sort_flag;
2429     NMEM rset_nmem=NULL;
2430
2431     *list = 0;
2432
2433     if (attributeset == VAL_NONE)
2434         attributeset = VAL_BIB1;
2435
2436     if (!limit_set)
2437     {
2438         AttrType termset;
2439         int termset_value_numeric;
2440         const char *termset_value_string;
2441         attr_init (&termset, zapt, 8);
2442         termset_value_numeric =
2443             attr_find_ex (&termset, NULL, &termset_value_string);
2444         if (termset_value_numeric != -1)
2445         {
2446             char resname[32];
2447             const char *termset_name = 0;
2448
2449             if (termset_value_numeric != -2)
2450             {
2451
2452                 sprintf (resname, "%d", termset_value_numeric);
2453                 termset_name = resname;
2454             }
2455             else
2456                 termset_name = termset_value_string;
2457
2458             limit_set = resultSetRef (zh, termset_name);
2459         }
2460     }
2461
2462     yaz_log (YLOG_DEBUG, "position = %d, num = %d set=%d",
2463              pos, num, attributeset);
2464
2465     attr_init (&use, zapt, 1);
2466     use_value = attr_find_ex (&use, &attributeset, &use_string);
2467
2468     if (zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2469                          rank_type, &complete_flag, &sort_flag))
2470     {
2471         *num_entries = 0;
2472         zh->errCode = 113;
2473         return ;
2474     }
2475     yaz_log (YLOG_DEBUG, "use_value = %d", use_value);
2476
2477     if (use_value == -1)
2478         use_value = 1016;
2479     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2480     {
2481         int r;
2482         attent attp;
2483         data1_local_attribute *local_attr;
2484
2485         if ((r=att_getentbyatt (zh, &attp, attributeset, use_value,
2486                                 use_string)))
2487         {
2488             yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2489                   attributeset, use_value);
2490             if (r == -1)
2491             {
2492                 char val_str[32];
2493                 sprintf (val_str, "%d", use_value);
2494                 errCode = 114;
2495                 errString = odr_strdup (stream, val_str);
2496             }
2497             else
2498                 errCode = 121;
2499             continue;
2500         }
2501         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2502         {
2503             zh->errString = basenames[base_no];
2504             zh->errCode = 109; /* Database unavailable */
2505             *num_entries = 0;
2506             return;
2507         }
2508         bases_ok++;
2509         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2510              local_attr = local_attr->next)
2511         {
2512             int ord;
2513
2514             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
2515                                          local_attr->local);
2516             if (ord > 0)
2517                 ords[ord_no++] = ord;
2518         }
2519     }
2520     if (!bases_ok && errCode)
2521     {
2522         zh->errCode = errCode;
2523         zh->errString = errString;
2524         *num_entries = 0;
2525     }
2526     if (ord_no == 0)
2527     {
2528         *num_entries = 0;
2529         return;
2530     }
2531     /* prepare dictionary scanning */
2532     before = pos-1;
2533     after = 1+num-pos;
2534     scan_info_array = (struct scan_info *)
2535         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2536     for (i = 0; i < ord_no; i++)
2537     {
2538         int j, prefix_len = 0;
2539         int before_tmp = before, after_tmp = after;
2540         struct scan_info *scan_info = scan_info_array + i;
2541         struct rpn_char_map_info rcmi;
2542
2543         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2544
2545         scan_info->before = before;
2546         scan_info->after = after;
2547         scan_info->odr = stream;
2548
2549         scan_info->list = (struct scan_info_entry *)
2550             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2551         for (j = 0; j<before+after; j++)
2552             scan_info->list[j].term = NULL;
2553
2554         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2555         termz[prefix_len++] = reg_id;
2556         termz[prefix_len] = 0;
2557         strcpy(scan_info->prefix, termz);
2558
2559         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id))
2560             return ;
2561
2562         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2563                   scan_info, scan_handle);
2564     }
2565     glist = (ZebraScanEntry *)
2566         odr_malloc(stream, (before+after)*sizeof(*glist));
2567
2568     rset_nmem = nmem_create();
2569
2570     /* consider terms after main term */
2571     for (i = 0; i < ord_no; i++)
2572         ptr[i] = before;
2573
2574     *is_partial = 0;
2575     for (i = 0; i<after; i++)
2576     {
2577         int j, j0 = -1;
2578         const char *mterm = NULL;
2579         const char *tst;
2580         RSET rset;
2581
2582         for (j = 0; j < ord_no; j++)
2583         {
2584             if (ptr[j] < before+after &&
2585                 (tst=scan_info_array[j].list[ptr[j]].term) &&
2586                 (!mterm || strcmp (tst, mterm) < 0))
2587             {
2588                 j0 = j;
2589                 mterm = tst;
2590             }
2591         }
2592         if (j0 == -1)
2593             break;
2594         scan_term_untrans (zh, stream->mem, reg_id,
2595                            &glist[i+before].term, mterm);
2596         rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2597                            glist[i+before].term, strlen(glist[i+before].term),
2598                            NULL, 0, zapt->term->which, rset_nmem,
2599                            key_it_ctrl,key_it_ctrl->scope);
2600         ptr[j0]++;
2601         for (j = j0+1; j<ord_no; j++)
2602         {
2603             if (ptr[j] < before+after &&
2604                 (tst=scan_info_array[j].list[ptr[j]].term) &&
2605                 !strcmp (tst, mterm))
2606             {
2607                 RSET rset2;
2608
2609                 rset2 =
2610                    rset_trunc(zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2611                                glist[i+before].term,
2612                                strlen(glist[i+before].term), NULL, 0,
2613                                zapt->term->which,rset_nmem,
2614                                key_it_ctrl, key_it_ctrl->scope);
2615                 rset = rsbool_create_or(rset_nmem,key_it_ctrl,
2616                                key_it_ctrl->scope, rset, rset2);
2617                 /* FIXME - Use a proper multi-or */
2618
2619                 ptr[j]++;
2620             }
2621         }
2622         if (limit_set)
2623             rset = rsbool_create_and(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
2624                             rset, rset_dup(limit_set));
2625         count_set (rset, &glist[i+before].occurrences);
2626         rset_delete (rset);
2627     }
2628     if (i < after)
2629     {
2630         *num_entries -= (after-i);
2631         *is_partial = 1;
2632     }
2633
2634     /* consider terms before main term */
2635     for (i = 0; i<ord_no; i++)
2636         ptr[i] = 0;
2637
2638     for (i = 0; i<before; i++)
2639     {
2640         int j, j0 = -1;
2641         const char *mterm = NULL;
2642         const char *tst;
2643         RSET rset;
2644
2645         for (j = 0; j <ord_no; j++)
2646         {
2647             if (ptr[j] < before &&
2648                 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
2649                 (!mterm || strcmp (tst, mterm) > 0))
2650             {
2651                 j0 = j;
2652                 mterm = tst;
2653             }
2654         }
2655         if (j0 == -1)
2656             break;
2657
2658         scan_term_untrans (zh, stream->mem, reg_id,
2659                            &glist[before-1-i].term, mterm);
2660
2661         rset = rset_trunc
2662                (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2663                 glist[before-1-i].term, strlen(glist[before-1-i].term),
2664                 NULL, 0, zapt->term->which,rset_nmem,
2665                 key_it_ctrl,key_it_ctrl->scope);
2666
2667         ptr[j0]++;
2668
2669         for (j = j0+1; j<ord_no; j++)
2670         {
2671             if (ptr[j] < before &&
2672                 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
2673                 !strcmp (tst, mterm))
2674             {
2675                 RSET rset2;
2676
2677                 rset2 = rset_trunc(
2678                     zh,
2679                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2680                     glist[before-1-i].term,
2681                     strlen(glist[before-1-i].term), NULL, 0,
2682                     zapt->term->which, rset_nmem,
2683                     key_it_ctrl, key_it_ctrl->scope);
2684                 rset = rsbool_create_and(rset_nmem,key_it_ctrl,
2685                                          key_it_ctrl->scope, rset, rset2);
2686                 /* FIXME - multi-and ?? */
2687                 ptr[j]++;
2688             }
2689         }
2690         if (limit_set)
2691             rset = rsbool_create_and(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
2692                             rset, rset_dup(limit_set));
2693         count_set (rset, &glist[before-1-i].occurrences);
2694         rset_delete (rset);
2695     }
2696     i = before-i;
2697     if (i)
2698     {
2699         *is_partial = 1;
2700         *position -= i;
2701         *num_entries -= i;
2702     }
2703
2704     nmem_destroy(rset_nmem);
2705     *list = glist + i;               /* list is set to first 'real' entry */
2706
2707     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2708           *position, *num_entries);
2709     if (zh->errCode)
2710         yaz_log(YLOG_DEBUG, "scan error: %d", zh->errCode);
2711 }
2712