index/zrpn.c

   1 /* $Id: zrpn.c,v 1.213 2006-05-17 17:46:45 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 typedef struct
  47 {
  48     int type;
  49     int major;
  50     int minor;
  51     Z_AttributesPlusTerm *zapt;
  52 } AttrType;
  53
  54 static int log_level_set = 0;
  55 static int log_level_rpn = 0;
  56
  57 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  58 {
  59     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  60     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  61 #if 0
  62     if (out && *out)
  63     {
  64         const char *outp = *out;
  65         yaz_log(YLOG_LOG, "---");
  66         while (*outp)
  67         {
  68             yaz_log(YLOG_LOG, "%02X", *outp);
  69             outp++;
  70         }
  71     }
  72 #endif
  73     return out;
  74 }
  75
  76 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  77                                   struct rpn_char_map_info *map_info)
  78 {
  79     map_info->zm = reg->zebra_maps;
  80     map_info->reg_type = reg_type;
  81     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  82 }
  83
  84 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  85                         const char **string_value)
  86 {
  87     int num_attributes;
  88
  89     num_attributes = src->zapt->attributes->num_attributes;
  90     while (src->major < num_attributes)
  91     {
  92         Z_AttributeElement *element;
  93
  94         element = src->zapt->attributes->attributes[src->major];
  95         if (src->type == *element->attributeType)
  96         {
  97             switch (element->which)
  98             {
  99             case Z_AttributeValue_numeric:
 100                 ++(src->major);
 101                 if (element->attributeSet && attributeSetP)
 102                 {
 103                     oident *attrset;
 104
 105                     attrset = oid_getentbyoid(element->attributeSet);
 106                     *attributeSetP = attrset->value;
 107                 }
 108                 return *element->value.numeric;
 109                 break;
 110             case Z_AttributeValue_complex:
 111                 if (src->minor >= element->value.complex->num_list)
 112                     break;
 113                 if (element->attributeSet && attributeSetP)
 114                 {
 115                     oident *attrset;
 116
 117                     attrset = oid_getentbyoid(element->attributeSet);
 118                     *attributeSetP = attrset->value;
 119                 }
 120                 if (element->value.complex->list[src->minor]->which ==
 121                     Z_StringOrNumeric_numeric)
 122                 {
 123                     ++(src->minor);
 124                     return
 125                         *element->value.complex->list[src->minor-1]->u.numeric;
 126                 }
 127                 else if (element->value.complex->list[src->minor]->which ==
 128                          Z_StringOrNumeric_string)
 129                 {
 130                     if (!string_value)
 131                         break;
 132                     ++(src->minor);
 133                     *string_value =
 134                         element->value.complex->list[src->minor-1]->u.string;
 135                     return -2;
 136                 }
 137                 else
 138                     break;
 139             default:
 140                 assert(0);
 141             }
 142         }
 143         ++(src->major);
 144     }
 145     return -1;
 146 }
 147
 148 static int attr_find(AttrType *src, oid_value *attributeSetP)
 149 {
 150     return attr_find_ex(src, attributeSetP, 0);
 151 }
 152
 153 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 154                        int type)
 155 {
 156     src->zapt = zapt;
 157     src->type = type;
 158     src->major = 0;
 159     src->minor = 0;
 160 }
 161
 162 #define TERM_COUNT
 163
 164 struct grep_info {
 165 #ifdef TERM_COUNT
 166     int *term_no;
 167 #endif
 168     ISAM_P *isam_p_buf;
 169     int isam_p_size;
 170     int isam_p_indx;
 171     ZebraHandle zh;
 172     int reg_type;
 173     ZebraSet termset;
 174 };
 175
 176 void zebra_term_untrans(ZebraHandle zh, int reg_type,
 177                         char *dst, const char *src)
 178 {
 179     int len = 0;
 180     while (*src)
 181     {
 182         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 183                                            reg_type, &src);
 184         if (!cp)
 185         {
 186             if (len < IT_MAX_WORD-1)
 187                 dst[len++] = *src;
 188             src++;
 189         }
 190         else
 191             while (*cp && len < IT_MAX_WORD-1)
 192                 dst[len++] = *cp++;
 193     }
 194     dst[len] = '\0';
 195 }
 196
 197 static void add_isam_p(const char *name, const char *info,
 198                        struct grep_info *p)
 199 {
 200     if (!log_level_set)
 201     {
 202         log_level_rpn = yaz_log_module_level("rpn");
 203         log_level_set = 1;
 204     }
 205     if (p->isam_p_indx == p->isam_p_size)
 206     {
 207         ISAM_P *new_isam_p_buf;
 208 #ifdef TERM_COUNT
 209         int *new_term_no;
 210 #endif
 211         p->isam_p_size = 2*p->isam_p_size + 100;
 212         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 213                                             p->isam_p_size);
 214         if (p->isam_p_buf)
 215         {
 216             memcpy(new_isam_p_buf, p->isam_p_buf,
 217                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 218             xfree(p->isam_p_buf);
 219         }
 220         p->isam_p_buf = new_isam_p_buf;
 221
 222 #ifdef TERM_COUNT
 223         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 224         if (p->term_no)
 225         {
 226             memcpy(new_term_no, p->isam_p_buf,
 227                     p->isam_p_indx * sizeof(*p->term_no));
 228             xfree(p->term_no);
 229         }
 230         p->term_no = new_term_no;
 231 #endif
 232     }
 233     assert(*info == sizeof(*p->isam_p_buf));
 234     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 235
 236 #if 1
 237     if (p->termset)
 238     {
 239         const char *db;
 240         int set, use;
 241         char term_tmp[IT_MAX_WORD];
 242         int ord = 0;
 243         int len = key_SU_decode (&ord, (const unsigned char *) name);
 244
 245         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 246         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 247         zebraExplain_lookup_ord (p->zh->reg->zei,
 248                                  ord, 0 /* index_type */, &db, &set, &use, 0);
 249         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 250
 251         resultSetAddTerm(p->zh, p->termset, name[len], db,
 252                          set, use, term_tmp);
 253     }
 254 #endif
 255     (p->isam_p_indx)++;
 256 }
 257
 258 static int grep_handle(char *name, const char *info, void *p)
 259 {
 260     add_isam_p(name, info, (struct grep_info *) p);
 261     return 0;
 262 }
 263
 264 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 265                     const char *ct1, const char *ct2, int first)
 266 {
 267     const char *s1, *s0 = *src;
 268     const char **map;
 269
 270     /* skip white space */
 271     while (*s0)
 272     {
 273         if (ct1 && strchr(ct1, *s0))
 274             break;
 275         if (ct2 && strchr(ct2, *s0))
 276             break;
 277         s1 = s0;
 278         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 279         if (**map != *CHR_SPACE)
 280             break;
 281         s0 = s1;
 282     }
 283     *src = s0;
 284     return *s0;
 285 }
 286
 287
 288 static void esc_str(char *out_buf, int out_size,
 289                     const char *in_buf, int in_size)
 290 {
 291     int k;
 292
 293     assert(out_buf);
 294     assert(in_buf);
 295     assert(out_size > 20);
 296     *out_buf = '\0';
 297     for (k = 0; k<in_size; k++)
 298     {
 299         int c = in_buf[k] & 0xff;
 300         int pc;
 301         if (c < 32 || c > 126)
 302             pc = '?';
 303         else
 304             pc = c;
 305         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 306         if (strlen(out_buf) > out_size-20)
 307         {
 308             strcat(out_buf, "..");
 309             break;
 310         }
 311     }
 312 }
 313
 314 #define REGEX_CHARS " []()|.*+?!"
 315
 316 /* term_100: handle term, where trunc = none(no operators at all) */
 317 static int term_100(ZebraMaps zebra_maps, int reg_type,
 318                     const char **src, char *dst, int space_split,
 319                     char *dst_term)
 320 {
 321     const char *s0;
 322     const char **map;
 323     int i = 0;
 324     int j = 0;
 325
 326     const char *space_start = 0;
 327     const char *space_end = 0;
 328
 329     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 330         return 0;
 331     s0 = *src;
 332     while (*s0)
 333     {
 334         const char *s1 = s0;
 335         int q_map_match = 0;
 336         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 337                                 &q_map_match);
 338         if (space_split)
 339         {
 340             if (**map == *CHR_SPACE)
 341                 break;
 342         }
 343         else  /* complete subfield only. */
 344         {
 345             if (**map == *CHR_SPACE)
 346             {   /* save space mapping for later  .. */
 347                 space_start = s1;
 348                 space_end = s0;
 349                 continue;
 350             }
 351             else if (space_start)
 352             {   /* reload last space */
 353                 while (space_start < space_end)
 354                 {
 355                     if (strchr(REGEX_CHARS, *space_start))
 356                         dst[i++] = '\\';
 357                     dst_term[j++] = *space_start;
 358                     dst[i++] = *space_start++;
 359                 }
 360                 /* and reset */
 361                 space_start = space_end = 0;
 362             }
 363         }
 364         /* add non-space char */
 365         memcpy(dst_term+j, s1, s0 - s1);
 366         j += (s0 - s1);
 367         if (!q_map_match)
 368         {
 369             while (s1 < s0)
 370             {
 371                 if (strchr(REGEX_CHARS, *s1))
 372                     dst[i++] = '\\';
 373                 dst[i++] = *s1++;
 374             }
 375         }
 376         else
 377         {
 378             char tmpbuf[80];
 379             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 380
 381             strcpy(dst + i, map[0]);
 382             i += strlen(map[0]);
 383         }
 384     }
 385     dst[i] = '\0';
 386     dst_term[j] = '\0';
 387     *src = s0;
 388     return i;
 389 }
 390
 391 /* term_101: handle term, where trunc = Process # */
 392 static int term_101(ZebraMaps zebra_maps, int reg_type,
 393                     const char **src, char *dst, int space_split,
 394                     char *dst_term)
 395 {
 396     const char *s0;
 397     const char **map;
 398     int i = 0;
 399     int j = 0;
 400
 401     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 402         return 0;
 403     s0 = *src;
 404     while (*s0)
 405     {
 406         if (*s0 == '#')
 407         {
 408             dst[i++] = '.';
 409             dst[i++] = '*';
 410             dst_term[j++] = *s0++;
 411         }
 412         else
 413         {
 414             const char *s1 = s0;
 415             int q_map_match = 0;
 416             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 417                                     &q_map_match);
 418             if (space_split && **map == *CHR_SPACE)
 419                 break;
 420
 421             /* add non-space char */
 422             memcpy(dst_term+j, s1, s0 - s1);
 423             j += (s0 - s1);
 424             if (!q_map_match)
 425             {
 426                 while (s1 < s0)
 427                 {
 428                     if (strchr(REGEX_CHARS, *s1))
 429                         dst[i++] = '\\';
 430                     dst[i++] = *s1++;
 431                 }
 432             }
 433             else
 434             {
 435                 char tmpbuf[80];
 436                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 437
 438                 strcpy(dst + i, map[0]);
 439                 i += strlen(map[0]);
 440             }
 441         }
 442     }
 443     dst[i] = '\0';
 444     dst_term[j++] = '\0';
 445     *src = s0;
 446     return i;
 447 }
 448
 449 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 450 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 451                     char *dst, int *errors, int space_split,
 452                     char *dst_term)
 453 {
 454     int i = 0;
 455     int j = 0;
 456     const char *s0;
 457     const char **map;
 458
 459     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 460         return 0;
 461     s0 = *src;
 462     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 463         isdigit(((const unsigned char *)s0)[1]))
 464     {
 465         *errors = s0[1] - '0';
 466         s0 += 3;
 467         if (*errors > 3)
 468             *errors = 3;
 469     }
 470     while (*s0)
 471     {
 472         if (strchr("^\\()[].*+?|-", *s0))
 473         {
 474             dst_term[j++] = *s0;
 475             dst[i++] = *s0++;
 476         }
 477         else
 478         {
 479             const char *s1 = s0;
 480             int q_map_match = 0;
 481             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 482                                     &q_map_match);
 483             if (space_split && **map == *CHR_SPACE)
 484                 break;
 485
 486             /* add non-space char */
 487             memcpy(dst_term+j, s1, s0 - s1);
 488             j += (s0 - s1);
 489             if (!q_map_match)
 490             {
 491                 while (s1 < s0)
 492                 {
 493                     if (strchr(REGEX_CHARS, *s1))
 494                         dst[i++] = '\\';
 495                     dst[i++] = *s1++;
 496                 }
 497             }
 498             else
 499             {
 500                 char tmpbuf[80];
 501                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 502
 503                 strcpy(dst + i, map[0]);
 504                 i += strlen(map[0]);
 505             }
 506         }
 507     }
 508     dst[i] = '\0';
 509     dst_term[j] = '\0';
 510     *src = s0;
 511
 512     return i;
 513 }
 514
 515 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 516 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 517                     char *dst, int space_split, char *dst_term)
 518 {
 519     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 520                     dst_term);
 521 }
 522
 523
 524 /* term_104: handle term, where trunc = Process # and ! */
 525 static int term_104(ZebraMaps zebra_maps, int reg_type,
 526                     const char **src, char *dst, int space_split,
 527                     char *dst_term)
 528 {
 529     const char *s0;
 530     const char **map;
 531     int i = 0;
 532     int j = 0;
 533
 534     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 535         return 0;
 536     s0 = *src;
 537     while (*s0)
 538     {
 539         if (*s0 == '?')
 540         {
 541             dst_term[j++] = *s0++;
 542             if (*s0 >= '0' && *s0 <= '9')
 543             {
 544                 int limit = 0;
 545                 while (*s0 >= '0' && *s0 <= '9')
 546                 {
 547                     limit = limit * 10 + (*s0 - '0');
 548                     dst_term[j++] = *s0++;
 549                 }
 550                 if (limit > 20)
 551                     limit = 20;
 552                 while (--limit >= 0)
 553                 {
 554                     dst[i++] = '.';
 555                     dst[i++] = '?';
 556                 }
 557             }
 558             else
 559             {
 560                 dst[i++] = '.';
 561                 dst[i++] = '*';
 562             }
 563         }
 564         else if (*s0 == '*')
 565         {
 566             dst[i++] = '.';
 567             dst[i++] = '*';
 568             dst_term[j++] = *s0++;
 569         }
 570         else if (*s0 == '#')
 571         {
 572             dst[i++] = '.';
 573             dst_term[j++] = *s0++;
 574         }
 575         else
 576         {
 577             const char *s1 = s0;
 578             int q_map_match = 0;
 579             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 580                                     &q_map_match);
 581             if (space_split && **map == *CHR_SPACE)
 582                 break;
 583
 584             /* add non-space char */
 585             memcpy(dst_term+j, s1, s0 - s1);
 586             j += (s0 - s1);
 587             if (!q_map_match)
 588             {
 589                 while (s1 < s0)
 590                 {
 591                     if (strchr(REGEX_CHARS, *s1))
 592                         dst[i++] = '\\';
 593                     dst[i++] = *s1++;
 594                 }
 595             }
 596             else
 597             {
 598                 char tmpbuf[80];
 599                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 600
 601                 strcpy(dst + i, map[0]);
 602                 i += strlen(map[0]);
 603             }
 604         }
 605     }
 606     dst[i] = '\0';
 607     dst_term[j++] = '\0';
 608     *src = s0;
 609     return i;
 610 }
 611
 612 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 613 static int term_105(ZebraMaps zebra_maps, int reg_type,
 614                     const char **src, char *dst, int space_split,
 615                     char *dst_term, int right_truncate)
 616 {
 617     const char *s0;
 618     const char **map;
 619     int i = 0;
 620     int j = 0;
 621
 622     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 623         return 0;
 624     s0 = *src;
 625     while (*s0)
 626     {
 627         if (*s0 == '*')
 628         {
 629             dst[i++] = '.';
 630             dst[i++] = '*';
 631             dst_term[j++] = *s0++;
 632         }
 633         else if (*s0 == '!')
 634         {
 635             dst[i++] = '.';
 636             dst_term[j++] = *s0++;
 637         }
 638         else
 639         {
 640             const char *s1 = s0;
 641             int q_map_match = 0;
 642             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 643                                     &q_map_match);
 644             if (space_split && **map == *CHR_SPACE)
 645                 break;
 646
 647             /* add non-space char */
 648             memcpy(dst_term+j, s1, s0 - s1);
 649             j += (s0 - s1);
 650             if (!q_map_match)
 651             {
 652                 while (s1 < s0)
 653                 {
 654                     if (strchr(REGEX_CHARS, *s1))
 655                         dst[i++] = '\\';
 656                     dst[i++] = *s1++;
 657                 }
 658             }
 659             else
 660             {
 661                 char tmpbuf[80];
 662                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 663
 664                 strcpy(dst + i, map[0]);
 665                 i += strlen(map[0]);
 666             }
 667         }
 668     }
 669     if (right_truncate)
 670     {
 671         dst[i++] = '.';
 672         dst[i++] = '*';
 673     }
 674     dst[i] = '\0';
 675
 676     dst_term[j++] = '\0';
 677     *src = s0;
 678     return i;
 679 }
 680
 681
 682 /* gen_regular_rel - generate regular expression from relation
 683  *  val:     border value (inclusive)
 684  *  islt:    1 if <=; 0 if >=.
 685  */
 686 static void gen_regular_rel(char *dst, int val, int islt)
 687 {
 688     int dst_p;
 689     int w, d, i;
 690     int pos = 0;
 691     char numstr[20];
 692
 693     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 694     if (val >= 0)
 695     {
 696         if (islt)
 697             strcpy(dst, "(-[0-9]+|(");
 698         else
 699             strcpy(dst, "((");
 700     }
 701     else
 702     {
 703         if (!islt)
 704         {
 705             strcpy(dst, "([0-9]+|-(");
 706             dst_p = strlen(dst);
 707             islt = 1;
 708         }
 709         else
 710         {
 711             strcpy(dst, "(-(");
 712             islt = 0;
 713         }
 714         val = -val;
 715     }
 716     dst_p = strlen(dst);
 717     sprintf(numstr, "%d", val);
 718     for (w = strlen(numstr); --w >= 0; pos++)
 719     {
 720         d = numstr[w];
 721         if (pos > 0)
 722         {
 723             if (islt)
 724             {
 725                 if (d == '0')
 726                     continue;
 727                 d--;
 728             }
 729             else
 730             {
 731                 if (d == '9')
 732                     continue;
 733                 d++;
 734             }
 735         }
 736
 737         strcpy(dst + dst_p, numstr);
 738         dst_p = strlen(dst) - pos - 1;
 739
 740         if (islt)
 741         {
 742             if (d != '0')
 743             {
 744                 dst[dst_p++] = '[';
 745                 dst[dst_p++] = '0';
 746                 dst[dst_p++] = '-';
 747                 dst[dst_p++] = d;
 748                 dst[dst_p++] = ']';
 749             }
 750             else
 751                 dst[dst_p++] = d;
 752         }
 753         else
 754         {
 755             if (d != '9')
 756             {
 757                 dst[dst_p++] = '[';
 758                 dst[dst_p++] = d;
 759                 dst[dst_p++] = '-';
 760                 dst[dst_p++] = '9';
 761                 dst[dst_p++] = ']';
 762             }
 763             else
 764                 dst[dst_p++] = d;
 765         }
 766         for (i = 0; i<pos; i++)
 767         {
 768             dst[dst_p++] = '[';
 769             dst[dst_p++] = '0';
 770             dst[dst_p++] = '-';
 771             dst[dst_p++] = '9';
 772             dst[dst_p++] = ']';
 773         }
 774         dst[dst_p++] = '|';
 775     }
 776     dst[dst_p] = '\0';
 777     if (islt)
 778     {
 779         /* match everything less than 10^(pos-1) */
 780         strcat(dst, "0*");
 781         for (i = 1; i<pos; i++)
 782             strcat(dst, "[0-9]?");
 783     }
 784     else
 785     {
 786         /* match everything greater than 10^pos */
 787         for (i = 0; i <= pos; i++)
 788             strcat(dst, "[0-9]");
 789         strcat(dst, "[0-9]*");
 790     }
 791     strcat(dst, "))");
 792 }
 793
 794 void string_rel_add_char(char **term_p, const char *src, int *indx)
 795 {
 796     if (src[*indx] == '\\')
 797         *(*term_p)++ = src[(*indx)++];
 798     *(*term_p)++ = src[(*indx)++];
 799 }
 800
 801 /*
 802  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 803  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 804  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 805  *              ([^-a].*|a[^-b].*|ab[c-].*)
 806  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 807  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 808  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 809  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 810  */
 811 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 812                            const char **term_sub, char *term_dict,
 813                            oid_value attributeSet,
 814                            int reg_type, int space_split, char *term_dst,
 815                            int *error_code)
 816 {
 817     AttrType relation;
 818     int relation_value;
 819     int i;
 820     char *term_tmp = term_dict + strlen(term_dict);
 821     char term_component[2*IT_MAX_WORD+20];
 822
 823     attr_init(&relation, zapt, 2);
 824     relation_value = attr_find(&relation, NULL);
 825
 826     *error_code = 0;
 827     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 828     switch (relation_value)
 829     {
 830     case 1:
 831         if (!term_100(zh->reg->zebra_maps, reg_type,
 832                       term_sub, term_component,
 833                       space_split, term_dst))
 834             return 0;
 835         yaz_log(log_level_rpn, "Relation <");
 836
 837         *term_tmp++ = '(';
 838         for (i = 0; term_component[i]; )
 839         {
 840             int j = 0;
 841
 842             if (i)
 843                 *term_tmp++ = '|';
 844             while (j < i)
 845                 string_rel_add_char(&term_tmp, term_component, &j);
 846
 847             *term_tmp++ = '[';
 848
 849             *term_tmp++ = '^';
 850             string_rel_add_char(&term_tmp, term_component, &i);
 851             *term_tmp++ = '-';
 852
 853             *term_tmp++ = ']';
 854             *term_tmp++ = '.';
 855             *term_tmp++ = '*';
 856
 857             if ((term_tmp - term_dict) > IT_MAX_WORD)
 858                 break;
 859         }
 860         *term_tmp++ = ')';
 861         *term_tmp = '\0';
 862         break;
 863     case 2:
 864         if (!term_100(zh->reg->zebra_maps, reg_type,
 865                       term_sub, term_component,
 866                       space_split, term_dst))
 867             return 0;
 868         yaz_log(log_level_rpn, "Relation <=");
 869
 870         *term_tmp++ = '(';
 871         for (i = 0; term_component[i]; )
 872         {
 873             int j = 0;
 874
 875             while (j < i)
 876                 string_rel_add_char(&term_tmp, term_component, &j);
 877             *term_tmp++ = '[';
 878
 879             *term_tmp++ = '^';
 880             string_rel_add_char(&term_tmp, term_component, &i);
 881             *term_tmp++ = '-';
 882
 883             *term_tmp++ = ']';
 884             *term_tmp++ = '.';
 885             *term_tmp++ = '*';
 886
 887             *term_tmp++ = '|';
 888
 889             if ((term_tmp - term_dict) > IT_MAX_WORD)
 890                 break;
 891         }
 892         for (i = 0; term_component[i]; )
 893             string_rel_add_char(&term_tmp, term_component, &i);
 894         *term_tmp++ = ')';
 895         *term_tmp = '\0';
 896         break;
 897     case 5:
 898         if (!term_100 (zh->reg->zebra_maps, reg_type,
 899                        term_sub, term_component, space_split, term_dst))
 900             return 0;
 901         yaz_log(log_level_rpn, "Relation >");
 902
 903         *term_tmp++ = '(';
 904         for (i = 0; term_component[i];)
 905         {
 906             int j = 0;
 907
 908             while (j < i)
 909                 string_rel_add_char(&term_tmp, term_component, &j);
 910             *term_tmp++ = '[';
 911
 912             *term_tmp++ = '^';
 913             *term_tmp++ = '-';
 914             string_rel_add_char(&term_tmp, term_component, &i);
 915
 916             *term_tmp++ = ']';
 917             *term_tmp++ = '.';
 918             *term_tmp++ = '*';
 919
 920             *term_tmp++ = '|';
 921
 922             if ((term_tmp - term_dict) > IT_MAX_WORD)
 923                 break;
 924         }
 925         for (i = 0; term_component[i];)
 926             string_rel_add_char(&term_tmp, term_component, &i);
 927         *term_tmp++ = '.';
 928         *term_tmp++ = '+';
 929         *term_tmp++ = ')';
 930         *term_tmp = '\0';
 931         break;
 932     case 4:
 933         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 934                       term_component, space_split, term_dst))
 935             return 0;
 936         yaz_log(log_level_rpn, "Relation >=");
 937
 938         *term_tmp++ = '(';
 939         for (i = 0; term_component[i];)
 940         {
 941             int j = 0;
 942
 943             if (i)
 944                 *term_tmp++ = '|';
 945             while (j < i)
 946                 string_rel_add_char(&term_tmp, term_component, &j);
 947             *term_tmp++ = '[';
 948
 949             if (term_component[i+1])
 950             {
 951                 *term_tmp++ = '^';
 952                 *term_tmp++ = '-';
 953                 string_rel_add_char(&term_tmp, term_component, &i);
 954             }
 955             else
 956             {
 957                 string_rel_add_char(&term_tmp, term_component, &i);
 958                 *term_tmp++ = '-';
 959             }
 960             *term_tmp++ = ']';
 961             *term_tmp++ = '.';
 962             *term_tmp++ = '*';
 963
 964             if ((term_tmp - term_dict) > IT_MAX_WORD)
 965                 break;
 966         }
 967         *term_tmp++ = ')';
 968         *term_tmp = '\0';
 969         break;
 970     case 3:
 971     case 102:
 972     case 103:
 973     case -1:
 974         if (!**term_sub)
 975             return 1;
 976         yaz_log(log_level_rpn, "Relation =");
 977         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 978                       term_component, space_split, term_dst))
 979             return 0;
 980         strcat(term_tmp, "(");
 981         strcat(term_tmp, term_component);
 982         strcat(term_tmp, ")");
 983         break;
 984     default:
 985         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 986         return 0;
 987     }
 988     return 1;
 989 }
 990
 991 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 992                              const char **term_sub,
 993                              oid_value attributeSet, NMEM stream,
 994                              struct grep_info *grep_info,
 995                              int reg_type, int complete_flag,
 996                              int num_bases, char **basenames,
 997                              char *term_dst, int xpath_use,
 998                              struct ord_list **ol);
 999
1000 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
1001                                  Z_AttributesPlusTerm *zapt,
1002                                  zint *hits_limit_value,
1003                                  const char **term_ref_id_str,
1004                                  NMEM nmem)
1005 {
1006     AttrType term_ref_id_attr;
1007     AttrType hits_limit_attr;
1008     int term_ref_id_int;
1009
1010     attr_init(&hits_limit_attr, zapt, 9);
1011     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
1012
1013     attr_init(&term_ref_id_attr, zapt, 10);
1014     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
1015     if (term_ref_id_int >= 0)
1016     {
1017         char *res = nmem_malloc(nmem, 20);
1018         sprintf(res, "%d", term_ref_id_int);
1019         *term_ref_id_str = res;
1020     }
1021
1022     /* no limit given ? */
1023     if (*hits_limit_value == -1)
1024     {
1025         if (*term_ref_id_str)
1026         {
1027             /* use global if term_ref is present */
1028             *hits_limit_value = zh->approx_limit;
1029         }
1030         else
1031         {
1032             /* no counting if term_ref is not present */
1033             *hits_limit_value = 0;
1034         }
1035     }
1036     else if (*hits_limit_value == 0)
1037     {
1038         /* 0 is the same as global limit */
1039         *hits_limit_value = zh->approx_limit;
1040     }
1041     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
1042             *term_ref_id_str ? *term_ref_id_str : "none",
1043             *hits_limit_value);
1044     return ZEBRA_OK;
1045 }
1046
1047 static ZEBRA_RES term_trunc(ZebraHandle zh,
1048                             Z_AttributesPlusTerm *zapt,
1049                             const char **term_sub,
1050                             oid_value attributeSet, NMEM stream,
1051                             struct grep_info *grep_info,
1052                             int reg_type, int complete_flag,
1053                             int num_bases, char **basenames,
1054                             char *term_dst,
1055                             const char *rank_type, int xpath_use,
1056                             NMEM rset_nmem,
1057                             RSET *rset,
1058                             struct rset_key_control *kc)
1059 {
1060     ZEBRA_RES res;
1061     struct ord_list *ol;
1062     zint hits_limit_value;
1063     const char *term_ref_id_str = 0;
1064     *rset = 0;
1065
1066     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str,
1067                     stream);
1068     grep_info->isam_p_indx = 0;
1069     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
1070                       reg_type, complete_flag, num_bases, basenames,
1071                       term_dst, xpath_use, &ol);
1072     if (res != ZEBRA_OK)
1073         return res;
1074     if (!*term_sub)  /* no more terms ? */
1075         return res;
1076     yaz_log(log_level_rpn, "term: %s", term_dst);
1077     *rset = rset_trunc(zh, grep_info->isam_p_buf,
1078                        grep_info->isam_p_indx, term_dst,
1079                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1080                        zapt->term->which, rset_nmem,
1081                        kc, kc->scope, ol, reg_type, hits_limit_value,
1082                        term_ref_id_str);
1083     if (!*rset)
1084         return ZEBRA_FAIL;
1085     return ZEBRA_OK;
1086 }
1087
1088 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1089                              const char **term_sub,
1090                              oid_value attributeSet, NMEM stream,
1091                              struct grep_info *grep_info,
1092                              int reg_type, int complete_flag,
1093                              int num_bases, char **basenames,
1094                              char *term_dst, int xpath_use,
1095                              struct ord_list **ol)
1096 {
1097     char term_dict[2*IT_MAX_WORD+4000];
1098     int j, r, base_no;
1099     AttrType truncation;
1100     int truncation_value;
1101     AttrType use;
1102     int use_value;
1103     const char *use_string = 0;
1104     oid_value curAttributeSet = attributeSet;
1105     const char *termp;
1106     struct rpn_char_map_info rcmi;
1107     int space_split = complete_flag ? 0 : 1;
1108
1109     int bases_ok = 0;     /* no of databases with OK attribute */
1110
1111     *ol = ord_list_create(stream);
1112
1113     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1114     attr_init(&use, zapt, 1);
1115     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1116     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1117     attr_init(&truncation, zapt, 5);
1118     truncation_value = attr_find(&truncation, NULL);
1119     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1120
1121     if (use_value == -1)    /* no attribute - assumy "any" */
1122         use_value = 1016;
1123     for (base_no = 0; base_no < num_bases; base_no++)
1124     {
1125         int ord = -1;
1126         int attr_ok = 0;
1127         int regex_range = 0;
1128         int init_pos = 0;
1129         attent attp;
1130         data1_local_attribute id_xpath_attr;
1131         data1_local_attribute *local_attr;
1132         int max_pos, prefix_len = 0;
1133         int relation_error;
1134
1135         termp = *term_sub;
1136
1137         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1138         {
1139             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1140                            basenames[base_no]);
1141             return ZEBRA_FAIL;
1142         }
1143         if (xpath_use > 0 && use_value == -2)
1144         {
1145             /* xpath mode and we have a string attribute */
1146             attp.local_attributes = &id_xpath_attr;
1147             attp.attset_ordinal = VAL_IDXPATH;
1148             id_xpath_attr.next = 0;
1149
1150             use_value = xpath_use;  /* xpath_use as use-attribute now */
1151             id_xpath_attr.local = use_value;
1152         }
1153         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1154         {
1155             /* X-Path attribute, use numeric value directly */
1156             attp.local_attributes = &id_xpath_attr;
1157             attp.attset_ordinal = VAL_IDXPATH;
1158             id_xpath_attr.next = 0;
1159             id_xpath_attr.local = use_value;
1160         }
1161         else if (use_string &&
1162                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1163                                                      reg_type,
1164                                                      use_string)) >= 0)
1165         {
1166             /* we have a match for a raw string attribute */
1167             char ord_buf[32];
1168             int i, ord_len;
1169
1170             if (prefix_len)
1171                 term_dict[prefix_len++] = '|';
1172             else
1173                 term_dict[prefix_len++] = '(';
1174
1175             ord_len = key_SU_encode (ord, ord_buf);
1176             for (i = 0; i<ord_len; i++)
1177             {
1178                 term_dict[prefix_len++] = 1;
1179                 term_dict[prefix_len++] = ord_buf[i];
1180             }
1181             attp.local_attributes = 0;  /* no more attributes */
1182             *ol = ord_list_append(stream, *ol, ord);
1183         }
1184         else
1185         {
1186             /* lookup in the .att files . Allow string as well */
1187             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1188                                       use_string)))
1189             {
1190                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1191                         curAttributeSet, use_value, r);
1192                 if (r == -1)
1193                 {
1194                     /* set was found, but value wasn't defined */
1195                     if (use_string)
1196                         zebra_setError(zh,
1197                                        YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1198                                        use_string);
1199                     else
1200                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1201                                             use_value);
1202                 }
1203                 else
1204                 {
1205                     int oid[OID_SIZE];
1206                     struct oident oident;
1207
1208                     oident.proto = PROTO_Z3950;
1209                     oident.oclass = CLASS_ATTSET;
1210                     oident.value = curAttributeSet;
1211                     oid_ent_to_oid (&oident, oid);
1212
1213                     zebra_setError(zh,
1214                                    YAZ_BIB1_UNSUPP_ATTRIBUTE_SET,
1215                                    oident.desc);
1216
1217                 }
1218                 continue;
1219             }
1220         }
1221         for (local_attr = attp.local_attributes; local_attr;
1222              local_attr = local_attr->next)
1223         {
1224             char ord_buf[32];
1225             int i, ord_len;
1226
1227             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1228                                               reg_type,
1229                                               attp.attset_ordinal,
1230                                               local_attr->local);
1231             if (ord < 0)
1232                 continue;
1233             *ol = ord_list_append(stream, *ol, ord);
1234             if (prefix_len)
1235                 term_dict[prefix_len++] = '|';
1236             else
1237                 term_dict[prefix_len++] = '(';
1238
1239             ord_len = key_SU_encode (ord, ord_buf);
1240             for (i = 0; i<ord_len; i++)
1241             {
1242                 term_dict[prefix_len++] = 1;
1243                 term_dict[prefix_len++] = ord_buf[i];
1244             }
1245             if (ord_len > init_pos)
1246                 init_pos = ord_len;
1247         }
1248         bases_ok++;
1249         if (prefix_len)
1250             attr_ok = 1;
1251
1252         term_dict[prefix_len++] = ')';
1253         term_dict[prefix_len] = '\0';
1254         j = prefix_len;
1255         switch (truncation_value)
1256         {
1257         case -1:         /* not specified */
1258         case 100:        /* do not truncate */
1259             if (!string_relation (zh, zapt, &termp, term_dict,
1260                                   attributeSet,
1261                                   reg_type, space_split, term_dst,
1262                                   &relation_error))
1263             {
1264                 if (relation_error)
1265                 {
1266                     zebra_setError(zh, relation_error, 0);
1267                     return ZEBRA_FAIL;
1268                 }
1269                 *term_sub = 0;
1270                 return ZEBRA_OK;
1271             }
1272             break;
1273         case 1:          /* right truncation */
1274             term_dict[j++] = '(';
1275             if (!term_100(zh->reg->zebra_maps, reg_type,
1276                           &termp, term_dict + j, space_split, term_dst))
1277             {
1278                 *term_sub = 0;
1279                 return ZEBRA_OK;
1280             }
1281             strcat(term_dict, ".*)");
1282             break;
1283         case 2:          /* keft truncation */
1284             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1285             if (!term_100(zh->reg->zebra_maps, reg_type,
1286                           &termp, term_dict + j, space_split, term_dst))
1287             {
1288                 *term_sub = 0;
1289                 return ZEBRA_OK;
1290             }
1291             strcat(term_dict, ")");
1292             break;
1293         case 3:          /* left&right truncation */
1294             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1295             if (!term_100(zh->reg->zebra_maps, reg_type,
1296                           &termp, term_dict + j, space_split, term_dst))
1297             {
1298                 *term_sub = 0;
1299                 return ZEBRA_OK;
1300             }
1301             strcat(term_dict, ".*)");
1302             break;
1303         case 101:        /* process # in term */
1304             term_dict[j++] = '(';
1305             if (!term_101(zh->reg->zebra_maps, reg_type,
1306                           &termp, term_dict + j, space_split, term_dst))
1307             {
1308                 *term_sub = 0;
1309                 return ZEBRA_OK;
1310             }
1311             strcat(term_dict, ")");
1312             break;
1313         case 102:        /* Regexp-1 */
1314             term_dict[j++] = '(';
1315             if (!term_102(zh->reg->zebra_maps, reg_type,
1316                           &termp, term_dict + j, space_split, term_dst))
1317             {
1318                 *term_sub = 0;
1319                 return ZEBRA_OK;
1320             }
1321             strcat(term_dict, ")");
1322             break;
1323         case 103:       /* Regexp-2 */
1324             regex_range = 1;
1325             term_dict[j++] = '(';
1326             if (!term_103(zh->reg->zebra_maps, reg_type,
1327                           &termp, term_dict + j, &regex_range,
1328                           space_split, term_dst))
1329             {
1330                 *term_sub = 0;
1331                 return ZEBRA_OK;
1332             }
1333             strcat(term_dict, ")");
1334             break;
1335         case 104:        /* process # and ! in term */
1336             term_dict[j++] = '(';
1337             if (!term_104(zh->reg->zebra_maps, reg_type,
1338                           &termp, term_dict + j, space_split, term_dst))
1339             {
1340                 *term_sub = 0;
1341                 return ZEBRA_OK;
1342             }
1343             strcat(term_dict, ")");
1344             break;
1345         case 105:        /* process * and ! in term */
1346             term_dict[j++] = '(';
1347             if (!term_105(zh->reg->zebra_maps, reg_type,
1348                           &termp, term_dict + j, space_split, term_dst, 1))
1349             {
1350                 *term_sub = 0;
1351                 return ZEBRA_OK;
1352             }
1353             strcat(term_dict, ")");
1354             break;
1355         case 106:        /* process * and ! in term */
1356             term_dict[j++] = '(';
1357             if (!term_105(zh->reg->zebra_maps, reg_type,
1358                           &termp, term_dict + j, space_split, term_dst, 0))
1359             {
1360                 *term_sub = 0;
1361                 return ZEBRA_OK;
1362             }
1363             strcat(term_dict, ")");
1364             break;
1365         default:
1366             zebra_setError_zint(zh,
1367                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1368                                 truncation_value);
1369             return ZEBRA_FAIL;
1370         }
1371         if (attr_ok)
1372         {
1373             char buf[80];
1374             const char *input = term_dict + prefix_len;
1375             esc_str(buf, sizeof(buf), input, strlen(input));
1376         }
1377         if (attr_ok)
1378         {
1379             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1380             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1381                                  grep_info, &max_pos, init_pos,
1382                                  grep_handle);
1383             if (r)
1384                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1385         }
1386     }
1387     if (!bases_ok)
1388         return ZEBRA_FAIL;
1389     *term_sub = termp;
1390     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1391     return ZEBRA_OK;
1392 }
1393
1394
1395 /* convert APT search term to UTF8 */
1396 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1397                                    char *termz)
1398 {
1399     size_t sizez;
1400     Z_Term *term = zapt->term;
1401
1402     switch (term->which)
1403     {
1404     case Z_Term_general:
1405         if (zh->iconv_to_utf8 != 0)
1406         {
1407             char *inbuf = (char *) term->u.general->buf;
1408             size_t inleft = term->u.general->len;
1409             char *outbuf = termz;
1410             size_t outleft = IT_MAX_WORD-1;
1411             size_t ret;
1412
1413             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1414                         &outbuf, &outleft);
1415             if (ret == (size_t)(-1))
1416             {
1417                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1418                 zebra_setError(
1419                     zh,
1420                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1421                     0);
1422                 return ZEBRA_FAIL;
1423             }
1424             *outbuf = 0;
1425         }
1426         else
1427         {
1428             sizez = term->u.general->len;
1429             if (sizez > IT_MAX_WORD-1)
1430                 sizez = IT_MAX_WORD-1;
1431             memcpy (termz, term->u.general->buf, sizez);
1432             termz[sizez] = '\0';
1433         }
1434         break;
1435     case Z_Term_characterString:
1436         sizez = strlen(term->u.characterString);
1437         if (sizez > IT_MAX_WORD-1)
1438             sizez = IT_MAX_WORD-1;
1439         memcpy (termz, term->u.characterString, sizez);
1440         termz[sizez] = '\0';
1441         break;
1442     default:
1443         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1444         return ZEBRA_FAIL;
1445     }
1446     return ZEBRA_OK;
1447 }
1448
1449 /* convert APT SCAN term to internal cmap */
1450 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1451                                  char *termz, int reg_type)
1452 {
1453     char termz0[IT_MAX_WORD];
1454
1455     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1456         return ZEBRA_FAIL;    /* error */
1457     else
1458     {
1459         const char **map;
1460         const char *cp = (const char *) termz0;
1461         const char *cp_end = cp + strlen(cp);
1462         const char *src;
1463         int i = 0;
1464         const char *space_map = NULL;
1465         int len;
1466
1467         while ((len = (cp_end - cp)) > 0)
1468         {
1469             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1470             if (**map == *CHR_SPACE)
1471                 space_map = *map;
1472             else
1473             {
1474                 if (i && space_map)
1475                     for (src = space_map; *src; src++)
1476                         termz[i++] = *src;
1477                 space_map = NULL;
1478                 for (src = *map; *src; src++)
1479                     termz[i++] = *src;
1480             }
1481         }
1482         termz[i] = '\0';
1483     }
1484     return ZEBRA_OK;
1485 }
1486
1487 static void grep_info_delete(struct grep_info *grep_info)
1488 {
1489 #ifdef TERM_COUNT
1490     xfree(grep_info->term_no);
1491 #endif
1492     xfree(grep_info->isam_p_buf);
1493 }
1494
1495 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1496                                    Z_AttributesPlusTerm *zapt,
1497                                    struct grep_info *grep_info,
1498                                    int reg_type)
1499 {
1500     AttrType termset;
1501     int termset_value_numeric;
1502     const char *termset_value_string;
1503
1504 #ifdef TERM_COUNT
1505     grep_info->term_no = 0;
1506 #endif
1507     grep_info->isam_p_size = 0;
1508     grep_info->isam_p_buf = NULL;
1509     grep_info->zh = zh;
1510     grep_info->reg_type = reg_type;
1511     grep_info->termset = 0;
1512
1513     if (!zapt)
1514         return ZEBRA_OK;
1515     attr_init(&termset, zapt, 8);
1516     termset_value_numeric =
1517         attr_find_ex(&termset, NULL, &termset_value_string);
1518     if (termset_value_numeric != -1)
1519     {
1520         char resname[32];
1521         const char *termset_name = 0;
1522         if (termset_value_numeric != -2)
1523         {
1524
1525             sprintf(resname, "%d", termset_value_numeric);
1526             termset_name = resname;
1527         }
1528         else
1529             termset_name = termset_value_string;
1530         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1531         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1532         if (!grep_info->termset)
1533         {
1534             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1535             return ZEBRA_FAIL;
1536         }
1537     }
1538     return ZEBRA_OK;
1539 }
1540
1541 /**
1542   \brief Create result set(s) for list of terms
1543   \param zh Zebra Handle
1544   \param termz term as used in query but converted to UTF-8
1545   \param attributeSet default attribute set
1546   \param stream memory for result
1547   \param reg_type register type ('w', 'p',..)
1548   \param complete_flag whether it's phrases or not
1549   \param rank_type term flags for ranking
1550   \param xpath_use use attribute for X-Path (-1 for no X-path)
1551   \param num_bases number of databases
1552   \param basenames array of databases
1553   \param rset_mem memory for result sets
1554   \param result_sets output result set for each term in list (output)
1555   \param number number of output result sets
1556   \param kc rset key control to be used for created result sets
1557 */
1558 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1559                                  Z_AttributesPlusTerm *zapt,
1560                                  const char *termz,
1561                                  oid_value attributeSet,
1562                                  NMEM stream,
1563                                  int reg_type, int complete_flag,
1564                                  const char *rank_type, int xpath_use,
1565                                  int num_bases, char **basenames,
1566                                  NMEM rset_nmem,
1567                                  RSET **result_sets, int *num_result_sets,
1568                                  struct rset_key_control *kc)
1569 {
1570     char term_dst[IT_MAX_WORD+1];
1571     struct grep_info grep_info;
1572     const char *termp = termz;
1573     int alloc_sets = 0;
1574     int empty_term = *termz ? 0 : 1;
1575
1576     empty_term = 0;
1577     *num_result_sets = 0;
1578     *term_dst = 0;
1579     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1580         return ZEBRA_FAIL;
1581     while(1)
1582     {
1583         ZEBRA_RES res;
1584
1585         if (alloc_sets == *num_result_sets)
1586         {
1587             int add = 10;
1588             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1589                                               sizeof(*rnew));
1590             if (alloc_sets)
1591                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1592             alloc_sets = alloc_sets + add;
1593             *result_sets = rnew;
1594         }
1595         res = term_trunc(zh, zapt, &termp, attributeSet,
1596                          stream, &grep_info,
1597                          reg_type, complete_flag,
1598                          num_bases, basenames,
1599                          term_dst, rank_type,
1600                          xpath_use, rset_nmem,
1601                          &(*result_sets)[*num_result_sets],
1602                          kc);
1603         if (res != ZEBRA_OK)
1604         {
1605             int i;
1606             for (i = 0; i < *num_result_sets; i++)
1607                 rset_delete((*result_sets)[i]);
1608             grep_info_delete (&grep_info);
1609             return res;
1610         }
1611         if ((*result_sets)[*num_result_sets] == 0)
1612             break;
1613         (*num_result_sets)++;
1614
1615         if (empty_term)
1616             break;
1617         if (!*termp)
1618             break;
1619     }
1620     grep_info_delete(&grep_info);
1621     return ZEBRA_OK;
1622 }
1623
1624 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1625                                        Z_AttributesPlusTerm *zapt,
1626                                        const char *termz_org,
1627                                        oid_value attributeSet,
1628                                        NMEM stream,
1629                                        int reg_type, int complete_flag,
1630                                        const char *rank_type, int xpath_use,
1631                                        int num_bases, char **basenames,
1632                                        NMEM rset_nmem,
1633                                        RSET *rset,
1634                                        struct rset_key_control *kc)
1635 {
1636     RSET *result_sets = 0;
1637     int num_result_sets = 0;
1638     ZEBRA_RES res =
1639         term_list_trunc(zh, zapt, termz_org, attributeSet,
1640                         stream, reg_type, complete_flag,
1641                         rank_type, xpath_use,
1642                         num_bases, basenames,
1643                         rset_nmem,
1644                         &result_sets, &num_result_sets, kc);
1645     if (res != ZEBRA_OK)
1646         return res;
1647     if (num_result_sets == 0)
1648         *rset = rsnull_create (rset_nmem, kc, 0);
1649     else if (num_result_sets == 1)
1650         *rset = result_sets[0];
1651     else
1652         *rset = rsprox_create(rset_nmem, kc, kc->scope,
1653                               num_result_sets, result_sets,
1654                               1 /* ordered */, 0 /* exclusion */,
1655                               3 /* relation */, 1 /* distance */);
1656     if (!*rset)
1657         return ZEBRA_FAIL;
1658     return ZEBRA_OK;
1659 }
1660
1661 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1662                                         Z_AttributesPlusTerm *zapt,
1663                                         const char *termz_org,
1664                                         oid_value attributeSet,
1665                                         NMEM stream,
1666                                         int reg_type, int complete_flag,
1667                                         const char *rank_type,
1668                                         int xpath_use,
1669                                         int num_bases, char **basenames,
1670                                         NMEM rset_nmem,
1671                                         RSET *rset,
1672                                         struct rset_key_control *kc)
1673 {
1674     RSET *result_sets = 0;
1675     int num_result_sets = 0;
1676     ZEBRA_RES res =
1677         term_list_trunc(zh, zapt, termz_org, attributeSet,
1678                         stream, reg_type, complete_flag,
1679                         rank_type, xpath_use,
1680                         num_bases, basenames,
1681                         rset_nmem,
1682                         &result_sets, &num_result_sets, kc);
1683     if (res != ZEBRA_OK)
1684         return res;
1685     if (num_result_sets == 0)
1686         *rset = rsnull_create (rset_nmem, kc, 0);
1687     else if (num_result_sets == 1)
1688         *rset = result_sets[0];
1689     else
1690         *rset = rsmulti_or_create(rset_nmem, kc, kc->scope, 0 /* termid */,
1691                                   num_result_sets, result_sets);
1692     if (!*rset)
1693         return ZEBRA_FAIL;
1694     return ZEBRA_OK;
1695 }
1696
1697 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1698                                          Z_AttributesPlusTerm *zapt,
1699                                          const char *termz_org,
1700                                          oid_value attributeSet,
1701                                          NMEM stream,
1702                                          int reg_type, int complete_flag,
1703                                          const char *rank_type,
1704                                          int xpath_use,
1705                                          int num_bases, char **basenames,
1706                                          NMEM rset_nmem,
1707                                          RSET *rset,
1708                                          struct rset_key_control *kc)
1709 {
1710     RSET *result_sets = 0;
1711     int num_result_sets = 0;
1712     ZEBRA_RES res =
1713         term_list_trunc(zh, zapt, termz_org, attributeSet,
1714                         stream, reg_type, complete_flag,
1715                         rank_type, xpath_use,
1716                         num_bases, basenames,
1717                         rset_nmem,
1718                         &result_sets, &num_result_sets,
1719                         kc);
1720     if (res != ZEBRA_OK)
1721         return res;
1722     if (num_result_sets == 0)
1723         *rset = rsnull_create (rset_nmem, kc, 0);
1724     else if (num_result_sets == 1)
1725         *rset = result_sets[0];
1726     else
1727         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1728                                    num_result_sets, result_sets);
1729     if (!*rset)
1730         return ZEBRA_FAIL;
1731     return ZEBRA_OK;
1732 }
1733
1734 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1735                             const char **term_sub,
1736                             char *term_dict,
1737                             oid_value attributeSet,
1738                             struct grep_info *grep_info,
1739                             int *max_pos,
1740                             int reg_type,
1741                             char *term_dst,
1742                             int *error_code)
1743 {
1744     AttrType relation;
1745     int relation_value;
1746     int term_value;
1747     int r;
1748     char *term_tmp = term_dict + strlen(term_dict);
1749
1750     *error_code = 0;
1751     attr_init(&relation, zapt, 2);
1752     relation_value = attr_find(&relation, NULL);
1753
1754     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1755
1756     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1757                   term_dst))
1758         return 0;
1759     term_value = atoi (term_tmp);
1760     switch (relation_value)
1761     {
1762     case 1:
1763         yaz_log(log_level_rpn, "Relation <");
1764         gen_regular_rel(term_tmp, term_value-1, 1);
1765         break;
1766     case 2:
1767         yaz_log(log_level_rpn, "Relation <=");
1768         gen_regular_rel(term_tmp, term_value, 1);
1769         break;
1770     case 4:
1771         yaz_log(log_level_rpn, "Relation >=");
1772         gen_regular_rel(term_tmp, term_value, 0);
1773         break;
1774     case 5:
1775         yaz_log(log_level_rpn, "Relation >");
1776         gen_regular_rel(term_tmp, term_value+1, 0);
1777         break;
1778     case -1:
1779     case 3:
1780         yaz_log(log_level_rpn, "Relation =");
1781         sprintf(term_tmp, "(0*%d)", term_value);
1782         break;
1783     default:
1784         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1785         return 0;
1786     }
1787     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1788     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1789                           0, grep_handle);
1790     if (r)
1791         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1792     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1793     return 1;
1794 }
1795
1796 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1797                               const char **term_sub,
1798                               oid_value attributeSet,
1799                               struct grep_info *grep_info,
1800                               int reg_type, int complete_flag,
1801                               int num_bases, char **basenames,
1802                               char *term_dst, int xpath_use, NMEM stream)
1803 {
1804     char term_dict[2*IT_MAX_WORD+2];
1805     int r, base_no;
1806     AttrType use;
1807     int use_value;
1808     const char *use_string = 0;
1809     oid_value curAttributeSet = attributeSet;
1810     const char *termp;
1811     struct rpn_char_map_info rcmi;
1812
1813     int bases_ok = 0;     /* no of databases with OK attribute */
1814
1815     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1816     attr_init(&use, zapt, 1);
1817     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1818
1819     if (use_value == -1)
1820         use_value = 1016;
1821
1822     for (base_no = 0; base_no < num_bases; base_no++)
1823     {
1824         attent attp;
1825         data1_local_attribute id_xpath_attr;
1826         data1_local_attribute *local_attr;
1827         int max_pos, prefix_len = 0;
1828         int relation_error = 0;
1829
1830         termp = *term_sub;
1831         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1832         {
1833             use_value = xpath_use;
1834             attp.local_attributes = &id_xpath_attr;
1835             attp.attset_ordinal = VAL_IDXPATH;
1836             id_xpath_attr.next = 0;
1837             id_xpath_attr.local = use_value;
1838         }
1839         else if (curAttributeSet == VAL_IDXPATH)
1840         {
1841             attp.local_attributes = &id_xpath_attr;
1842             attp.attset_ordinal = VAL_IDXPATH;
1843             id_xpath_attr.next = 0;
1844             id_xpath_attr.local = use_value;
1845         }
1846         else
1847         {
1848             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1849                                       use_string)))
1850             {
1851                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1852                       curAttributeSet, use_value, r);
1853                 if (r == -1)
1854                 {
1855                     if (use_string)
1856                         zebra_setError(zh,
1857                                        YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1858                                        use_string);
1859                     else
1860                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1861                                             use_value);
1862                 }
1863                 else
1864                     zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_SET, 0);
1865                 continue;
1866             }
1867         }
1868         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1869         {
1870             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1871                            basenames[base_no]);
1872             return ZEBRA_FAIL;
1873         }
1874         for (local_attr = attp.local_attributes; local_attr;
1875              local_attr = local_attr->next)
1876         {
1877             int ord;
1878             char ord_buf[32];
1879             int i, ord_len;
1880
1881             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1882                                               reg_type,
1883                                               attp.attset_ordinal,
1884                                               local_attr->local);
1885             if (ord < 0)
1886                 continue;
1887             if (prefix_len)
1888                 term_dict[prefix_len++] = '|';
1889             else
1890                 term_dict[prefix_len++] = '(';
1891
1892             ord_len = key_SU_encode (ord, ord_buf);
1893             for (i = 0; i<ord_len; i++)
1894             {
1895                 term_dict[prefix_len++] = 1;
1896                 term_dict[prefix_len++] = ord_buf[i];
1897             }
1898         }
1899         if (!prefix_len)
1900         {
1901             zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE, use_value);
1902             continue;
1903         }
1904         bases_ok++;
1905         term_dict[prefix_len++] = ')';
1906         term_dict[prefix_len] = '\0';
1907         if (!numeric_relation(zh, zapt, &termp, term_dict,
1908                               attributeSet, grep_info, &max_pos, reg_type,
1909                               term_dst, &relation_error))
1910         {
1911             if (relation_error)
1912             {
1913                 zebra_setError(zh, relation_error, 0);
1914                 return ZEBRA_FAIL;
1915             }
1916             *term_sub = 0;
1917             return ZEBRA_OK;
1918         }
1919     }
1920     if (!bases_ok)
1921         return ZEBRA_FAIL;
1922     *term_sub = termp;
1923     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1924     return ZEBRA_OK;
1925 }
1926
1927
1928 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1929                                         Z_AttributesPlusTerm *zapt,
1930                                         const char *termz,
1931                                         oid_value attributeSet,
1932                                         NMEM stream,
1933                                         int reg_type, int complete_flag,
1934                                         const char *rank_type, int xpath_use,
1935                                         int num_bases, char **basenames,
1936                                         NMEM rset_nmem,
1937                                         RSET *rset,
1938                                         struct rset_key_control *kc)
1939 {
1940     char term_dst[IT_MAX_WORD+1];
1941     const char *termp = termz;
1942     RSET *result_sets = 0;
1943     int num_result_sets = 0;
1944     ZEBRA_RES res;
1945     struct grep_info grep_info;
1946     int alloc_sets = 0;
1947     zint hits_limit_value;
1948     const char *term_ref_id_str = 0;
1949
1950     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1951
1952     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1953     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1954         return ZEBRA_FAIL;
1955     while (1)
1956     {
1957         if (alloc_sets == num_result_sets)
1958         {
1959             int add = 10;
1960             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1961                                               sizeof(*rnew));
1962             if (alloc_sets)
1963                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1964             alloc_sets = alloc_sets + add;
1965             result_sets = rnew;
1966         }
1967         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1968         grep_info.isam_p_indx = 0;
1969         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1970                            reg_type, complete_flag, num_bases, basenames,
1971                            term_dst, xpath_use,
1972                            stream);
1973         if (res == ZEBRA_FAIL || termp == 0)
1974             break;
1975         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1976         result_sets[num_result_sets] =
1977             rset_trunc(zh, grep_info.isam_p_buf,
1978                        grep_info.isam_p_indx, term_dst,
1979                        strlen(term_dst), rank_type,
1980                        0 /* preserve position */,
1981                        zapt->term->which, rset_nmem,
1982                        kc, kc->scope, 0, reg_type,
1983                        hits_limit_value,
1984                        term_ref_id_str);
1985         if (!result_sets[num_result_sets])
1986             break;
1987         num_result_sets++;
1988     }
1989     grep_info_delete(&grep_info);
1990     if (termp)
1991     {
1992         int i;
1993         for (i = 0; i<num_result_sets; i++)
1994             rset_delete(result_sets[i]);
1995         return ZEBRA_FAIL;
1996     }
1997     if (num_result_sets == 0)
1998         *rset = rsnull_create(rset_nmem, kc, 0);
1999     if (num_result_sets == 1)
2000         *rset = result_sets[0];
2001     else
2002         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
2003                                    num_result_sets, result_sets);
2004     if (!*rset)
2005         return ZEBRA_FAIL;
2006     return ZEBRA_OK;
2007 }
2008
2009 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
2010                                       Z_AttributesPlusTerm *zapt,
2011                                       const char *termz,
2012                                       oid_value attributeSet,
2013                                       NMEM stream,
2014                                       const char *rank_type, NMEM rset_nmem,
2015                                       RSET *rset,
2016                                       struct rset_key_control *kc)
2017 {
2018     RSFD rsfd;
2019     struct it_key key;
2020     int sys;
2021     *rset = rstemp_create(rset_nmem, kc, kc->scope,
2022                           res_get (zh->res, "setTmpDir"),0 );
2023     rsfd = rset_open(*rset, RSETF_WRITE);
2024
2025     sys = atoi(termz);
2026     if (sys <= 0)
2027         sys = 1;
2028     key.mem[0] = sys;
2029     key.mem[1] = 1;
2030     key.len = 2;
2031     rset_write (rsfd, &key);
2032     rset_close (rsfd);
2033     return ZEBRA_OK;
2034 }
2035
2036 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2037                                oid_value attributeSet, NMEM stream,
2038                                Z_SortKeySpecList *sort_sequence,
2039                                const char *rank_type,
2040                                NMEM rset_nmem,
2041                                RSET *rset,
2042                                struct rset_key_control *kc)
2043 {
2044     int i;
2045     int sort_relation_value;
2046     AttrType sort_relation_type;
2047     Z_SortKeySpec *sks;
2048     Z_SortKey *sk;
2049     int oid[OID_SIZE];
2050     oident oe;
2051     char termz[20];
2052
2053     attr_init(&sort_relation_type, zapt, 7);
2054     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
2055
2056     if (!sort_sequence->specs)
2057     {
2058         sort_sequence->num_specs = 10;
2059         sort_sequence->specs = (Z_SortKeySpec **)
2060             nmem_malloc(stream, sort_sequence->num_specs *
2061                          sizeof(*sort_sequence->specs));
2062         for (i = 0; i<sort_sequence->num_specs; i++)
2063             sort_sequence->specs[i] = 0;
2064     }
2065     if (zapt->term->which != Z_Term_general)
2066         i = 0;
2067     else
2068         i = atoi_n ((char *) zapt->term->u.general->buf,
2069                     zapt->term->u.general->len);
2070     if (i >= sort_sequence->num_specs)
2071         i = 0;
2072     sprintf(termz, "%d", i);
2073
2074     oe.proto = PROTO_Z3950;
2075     oe.oclass = CLASS_ATTSET;
2076     oe.value = attributeSet;
2077     if (!oid_ent_to_oid (&oe, oid))
2078         return ZEBRA_FAIL;
2079
2080     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2081     sks->sortElement = (Z_SortElement *)
2082         nmem_malloc(stream, sizeof(*sks->sortElement));
2083     sks->sortElement->which = Z_SortElement_generic;
2084     sk = sks->sortElement->u.generic = (Z_SortKey *)
2085         nmem_malloc(stream, sizeof(*sk));
2086     sk->which = Z_SortKey_sortAttributes;
2087     sk->u.sortAttributes = (Z_SortAttributes *)
2088         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2089
2090     sk->u.sortAttributes->id = oid;
2091     sk->u.sortAttributes->list = zapt->attributes;
2092
2093     sks->sortRelation = (int *)
2094         nmem_malloc(stream, sizeof(*sks->sortRelation));
2095     if (sort_relation_value == 1)
2096         *sks->sortRelation = Z_SortKeySpec_ascending;
2097     else if (sort_relation_value == 2)
2098         *sks->sortRelation = Z_SortKeySpec_descending;
2099     else
2100         *sks->sortRelation = Z_SortKeySpec_ascending;
2101
2102     sks->caseSensitivity = (int *)
2103         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2104     *sks->caseSensitivity = 0;
2105
2106     sks->which = Z_SortKeySpec_null;
2107     sks->u.null = odr_nullval ();
2108     sort_sequence->specs[i] = sks;
2109     *rset = rsnull_create (rset_nmem, kc, 0);
2110     return ZEBRA_OK;
2111 }
2112
2113
2114 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2115                        oid_value attributeSet,
2116                        struct xpath_location_step *xpath, int max, NMEM mem)
2117 {
2118     oid_value curAttributeSet = attributeSet;
2119     AttrType use;
2120     const char *use_string = 0;
2121
2122     attr_init(&use, zapt, 1);
2123     attr_find_ex(&use, &curAttributeSet, &use_string);
2124
2125     if (!use_string || *use_string != '/')
2126         return -1;
2127
2128     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2129 }
2130
2131
2132
2133 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2134                         int reg_type, const char *term, int use,
2135                         oid_value curAttributeSet, NMEM rset_nmem,
2136                         struct rset_key_control *kc)
2137 {
2138     RSET rset;
2139     struct grep_info grep_info;
2140     char term_dict[2048];
2141     char ord_buf[32];
2142     int prefix_len = 0;
2143     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, reg_type,
2144                                           curAttributeSet, use);
2145     int ord_len, i, r, max_pos;
2146     int term_type = Z_Term_characterString;
2147     const char *flags = "void";
2148
2149     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
2150         return rsnull_create(rset_nmem, kc, 0);
2151
2152     if (ord < 0)
2153         return rsnull_create(rset_nmem, kc, 0);
2154     if (prefix_len)
2155         term_dict[prefix_len++] = '|';
2156     else
2157         term_dict[prefix_len++] = '(';
2158
2159     ord_len = key_SU_encode (ord, ord_buf);
2160     for (i = 0; i<ord_len; i++)
2161     {
2162         term_dict[prefix_len++] = 1;
2163         term_dict[prefix_len++] = ord_buf[i];
2164     }
2165     term_dict[prefix_len++] = ')';
2166     strcpy(term_dict+prefix_len, term);
2167
2168     grep_info.isam_p_indx = 0;
2169     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2170                           &grep_info, &max_pos, 0, grep_handle);
2171     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2172              grep_info.isam_p_indx);
2173     rset = rset_trunc(zh, grep_info.isam_p_buf,
2174                       grep_info.isam_p_indx, term, strlen(term),
2175                       flags, 1, term_type,rset_nmem,
2176                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
2177                       0 /* term_ref_id_str */);
2178     grep_info_delete(&grep_info);
2179     return rset;
2180 }
2181
2182 static
2183 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
2184                            oid_value attributeSet,
2185                            int num_bases, char **basenames,
2186                            NMEM stream, const char *rank_type, RSET rset,
2187                            int xpath_len, struct xpath_location_step *xpath,
2188                            NMEM rset_nmem,
2189                            RSET *rset_out,
2190                            struct rset_key_control *kc)
2191 {
2192     oid_value curAttributeSet = attributeSet;
2193     int base_no;
2194     int i;
2195
2196     if (xpath_len < 0)
2197     {
2198         *rset_out = rset;
2199         return ZEBRA_OK;
2200     }
2201
2202     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2203     for (i = 0; i<xpath_len; i++)
2204     {
2205         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2206
2207     }
2208
2209     curAttributeSet = VAL_IDXPATH;
2210
2211     /*
2212       //a    ->    a/.*
2213       //a/b  ->    b/a/.*
2214       /a     ->    a/
2215       /a/b   ->    b/a/
2216
2217       /      ->    none
2218
2219    a[@attr = value]/b[@other = othervalue]
2220
2221  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2222  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2223  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2224  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2225  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2226  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2227
2228     */
2229
2230     dict_grep_cmap (zh->reg->dict, 0, 0);
2231
2232     for (base_no = 0; base_no < num_bases; base_no++)
2233     {
2234         int level = xpath_len;
2235         int first_path = 1;
2236
2237         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2238         {
2239             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2240                            basenames[base_no]);
2241             *rset_out = rset;
2242             return ZEBRA_FAIL;
2243         }
2244         while (--level >= 0)
2245         {
2246             char xpath_rev[128];
2247             int i, len;
2248             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2249
2250             *xpath_rev = 0;
2251             len = 0;
2252             for (i = level; i >= 1; --i)
2253             {
2254                 const char *cp = xpath[i].part;
2255                 if (*cp)
2256                 {
2257                     for (;*cp; cp++)
2258                         if (*cp == '*')
2259                         {
2260                             memcpy (xpath_rev + len, "[^/]*", 5);
2261                             len += 5;
2262                         }
2263                         else if (*cp == ' ')
2264                         {
2265
2266                             xpath_rev[len++] = 1;
2267                             xpath_rev[len++] = ' ';
2268                         }
2269
2270                         else
2271                             xpath_rev[len++] = *cp;
2272                     xpath_rev[len++] = '/';
2273                 }
2274                 else if (i == 1)  /* // case */
2275                 {
2276                     xpath_rev[len++] = '.';
2277                     xpath_rev[len++] = '*';
2278                 }
2279             }
2280             xpath_rev[len] = 0;
2281
2282             if (xpath[level].predicate &&
2283                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2284                 xpath[level].predicate->u.relation.name[0])
2285             {
2286                 WRBUF wbuf = wrbuf_alloc();
2287                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2288                 if (xpath[level].predicate->u.relation.value)
2289                 {
2290                     const char *cp = xpath[level].predicate->u.relation.value;
2291                     wrbuf_putc(wbuf, '=');
2292
2293                     while (*cp)
2294                     {
2295                         if (strchr(REGEX_CHARS, *cp))
2296                             wrbuf_putc(wbuf, '\\');
2297                         wrbuf_putc(wbuf, *cp);
2298                         cp++;
2299                     }
2300                 }
2301                 wrbuf_puts(wbuf, "");
2302                 rset_attr = xpath_trunc(
2303                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2304                     curAttributeSet, rset_nmem, kc);
2305                 wrbuf_free(wbuf, 1);
2306             }
2307             else
2308             {
2309                 if (!first_path)
2310                     continue;
2311             }
2312             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2313             if (strlen(xpath_rev))
2314             {
2315                 rset_start_tag = xpath_trunc(zh, stream, '0',
2316                         xpath_rev, 1, curAttributeSet, rset_nmem, kc);
2317
2318                 rset_end_tag = xpath_trunc(zh, stream, '0',
2319                         xpath_rev, 2, curAttributeSet, rset_nmem, kc);
2320
2321                 rset = rsbetween_create(rset_nmem, kc, kc->scope,
2322                                         rset_start_tag, rset,
2323                                         rset_end_tag, rset_attr);
2324             }
2325             first_path = 0;
2326         }
2327     }
2328     *rset_out = rset;
2329     return ZEBRA_OK;
2330 }
2331
2332 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2333                                 oid_value attributeSet, NMEM stream,
2334                                 Z_SortKeySpecList *sort_sequence,
2335                                 int num_bases, char **basenames,
2336                                 NMEM rset_nmem,
2337                                 RSET *rset,
2338                                 struct rset_key_control *kc)
2339 {
2340     ZEBRA_RES res = ZEBRA_OK;
2341     unsigned reg_id;
2342     char *search_type = NULL;
2343     char rank_type[128];
2344     int complete_flag;
2345     int sort_flag;
2346     char termz[IT_MAX_WORD+1];
2347     int xpath_len;
2348     int xpath_use = 0;
2349     struct xpath_location_step xpath[10];
2350
2351     if (!log_level_set)
2352     {
2353         log_level_rpn = yaz_log_module_level("rpn");
2354         log_level_set = 1;
2355     }
2356     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2357                     rank_type, &complete_flag, &sort_flag);
2358
2359     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2360     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2361     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2362     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2363
2364     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2365         return ZEBRA_FAIL;
2366
2367     if (sort_flag)
2368         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2369                              rank_type, rset_nmem, rset, kc);
2370     /* consider if an X-Path query is used */
2371     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2372     if (xpath_len >= 0)
2373     {
2374         xpath_use = 1016;  /* searching for element by default */
2375         if (xpath[xpath_len-1].part[0] == '@')
2376             xpath_use = 1015;  /* last step an attribute .. */
2377     }
2378
2379     /* search using one of the various search type strategies
2380        termz is our UTF-8 search term
2381        attributeSet is top-level default attribute set
2382        stream is ODR for search
2383        reg_id is the register type
2384        complete_flag is 1 for complete subfield, 0 for incomplete
2385        xpath_use is use-attribute to be used for X-Path search, 0 for none
2386     */
2387     if (!strcmp(search_type, "phrase"))
2388     {
2389         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2390                                     reg_id, complete_flag, rank_type,
2391                                     xpath_use,
2392                                     num_bases, basenames, rset_nmem,
2393                                     rset, kc);
2394     }
2395     else if (!strcmp(search_type, "and-list"))
2396     {
2397         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2398                                       reg_id, complete_flag, rank_type,
2399                                       xpath_use,
2400                                       num_bases, basenames, rset_nmem,
2401                                       rset, kc);
2402     }
2403     else if (!strcmp(search_type, "or-list"))
2404     {
2405         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2406                                      reg_id, complete_flag, rank_type,
2407                                      xpath_use,
2408                                      num_bases, basenames, rset_nmem,
2409                                      rset, kc);
2410     }
2411     else if (!strcmp(search_type, "local"))
2412     {
2413         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2414                                    rank_type, rset_nmem, rset, kc);
2415     }
2416     else if (!strcmp(search_type, "numeric"))
2417     {
2418         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2419                                      reg_id, complete_flag, rank_type,
2420                                      xpath_use,
2421                                      num_bases, basenames, rset_nmem,
2422                                      rset, kc);
2423     }
2424     else if (!strcmp(search_type, "always"))
2425     {
2426         *termz = '\0';
2427         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2428                                     reg_id, complete_flag, rank_type,
2429                                     xpath_use,
2430                                     num_bases, basenames, rset_nmem,
2431                                     rset, kc);
2432     }
2433     else
2434     {
2435         zebra_setError(zh, YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE, 0);
2436         res = ZEBRA_FAIL;
2437     }
2438     if (res != ZEBRA_OK)
2439         return res;
2440     if (!*rset)
2441         return ZEBRA_FAIL;
2442     return rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2443                             stream, rank_type, *rset,
2444                             xpath_len, xpath, rset_nmem, rset, kc);
2445 }
2446
2447 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2448                                       oid_value attributeSet,
2449                                       NMEM stream, NMEM rset_nmem,
2450                                       Z_SortKeySpecList *sort_sequence,
2451                                       int num_bases, char **basenames,
2452                                       RSET **result_sets, int *num_result_sets,
2453                                       Z_Operator *parent_op,
2454                                       struct rset_key_control *kc);
2455
2456 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2457                          oid_value attributeSet,
2458                          NMEM stream, NMEM rset_nmem,
2459                          Z_SortKeySpecList *sort_sequence,
2460                          int num_bases, char **basenames,
2461                          RSET *result_set)
2462 {
2463     RSET *result_sets = 0;
2464     int num_result_sets = 0;
2465     ZEBRA_RES res;
2466     struct rset_key_control *kc = zebra_key_control_create(zh);
2467
2468     res = rpn_search_structure(zh, zs, attributeSet,
2469                                stream, rset_nmem,
2470                                sort_sequence,
2471                                num_bases, basenames,
2472                                &result_sets, &num_result_sets,
2473                                0 /* no parent op */,
2474                                kc);
2475     if (res != ZEBRA_OK)
2476     {
2477         int i;
2478         for (i = 0; i<num_result_sets; i++)
2479             rset_delete(result_sets[i]);
2480         *result_set = 0;
2481     }
2482     else
2483     {
2484         assert(num_result_sets == 1);
2485         assert(result_sets);
2486         assert(*result_sets);
2487         *result_set = *result_sets;
2488     }
2489     (*kc->dec)(kc);
2490     return res;
2491 }
2492
2493 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2494                                oid_value attributeSet,
2495                                NMEM stream, NMEM rset_nmem,
2496                                Z_SortKeySpecList *sort_sequence,
2497                                int num_bases, char **basenames,
2498                                RSET **result_sets, int *num_result_sets,
2499                                Z_Operator *parent_op,
2500                                struct rset_key_control *kc)
2501 {
2502     *num_result_sets = 0;
2503     if (zs->which == Z_RPNStructure_complex)
2504     {
2505         ZEBRA_RES res;
2506         Z_Operator *zop = zs->u.complex->roperator;
2507         RSET *result_sets_l = 0;
2508         int num_result_sets_l = 0;
2509         RSET *result_sets_r = 0;
2510         int num_result_sets_r = 0;
2511
2512         res = rpn_search_structure(zh, zs->u.complex->s1,
2513                                    attributeSet, stream, rset_nmem,
2514                                    sort_sequence,
2515                                    num_bases, basenames,
2516                                    &result_sets_l, &num_result_sets_l,
2517                                    zop, kc);
2518         if (res != ZEBRA_OK)
2519         {
2520             int i;
2521             for (i = 0; i<num_result_sets_l; i++)
2522                 rset_delete(result_sets_l[i]);
2523             return res;
2524         }
2525         res = rpn_search_structure(zh, zs->u.complex->s2,
2526                                    attributeSet, stream, rset_nmem,
2527                                    sort_sequence,
2528                                    num_bases, basenames,
2529                                    &result_sets_r, &num_result_sets_r,
2530                                    zop, kc);
2531         if (res != ZEBRA_OK)
2532         {
2533             int i;
2534             for (i = 0; i<num_result_sets_l; i++)
2535                 rset_delete(result_sets_l[i]);
2536             for (i = 0; i<num_result_sets_r; i++)
2537                 rset_delete(result_sets_r[i]);
2538             return res;
2539         }
2540
2541         /* make a new list of result for all children */
2542         *num_result_sets = num_result_sets_l + num_result_sets_r;
2543         *result_sets = nmem_malloc(stream, *num_result_sets *
2544                                    sizeof(**result_sets));
2545         memcpy(*result_sets, result_sets_l,
2546                num_result_sets_l * sizeof(**result_sets));
2547         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2548                num_result_sets_r * sizeof(**result_sets));
2549
2550         if (!parent_op || parent_op->which != zop->which
2551             || (zop->which != Z_Operator_and &&
2552                 zop->which != Z_Operator_or))
2553         {
2554             /* parent node different from this one (or non-present) */
2555             /* we must combine result sets now */
2556             RSET rset;
2557             switch (zop->which)
2558             {
2559             case Z_Operator_and:
2560                 rset = rsmulti_and_create(rset_nmem, kc,
2561                                           kc->scope,
2562                                           *num_result_sets, *result_sets);
2563                 break;
2564             case Z_Operator_or:
2565                 rset = rsmulti_or_create(rset_nmem, kc,
2566                                          kc->scope, 0, /* termid */
2567                                          *num_result_sets, *result_sets);
2568                 break;
2569             case Z_Operator_and_not:
2570                 rset = rsbool_create_not(rset_nmem, kc,
2571                                          kc->scope,
2572                                          (*result_sets)[0],
2573                                          (*result_sets)[1]);
2574                 break;
2575             case Z_Operator_prox:
2576                 if (zop->u.prox->which != Z_ProximityOperator_known)
2577                 {
2578                     zebra_setError(zh,
2579                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2580                                    0);
2581                     return ZEBRA_FAIL;
2582                 }
2583                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2584                 {
2585                     zebra_setError_zint(zh,
2586                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2587                                         *zop->u.prox->u.known);
2588                     return ZEBRA_FAIL;
2589                 }
2590                 else
2591                 {
2592                     rset = rsprox_create(rset_nmem, kc,
2593                                          kc->scope,
2594                                          *num_result_sets, *result_sets,
2595                                          *zop->u.prox->ordered,
2596                                          (!zop->u.prox->exclusion ?
2597                                           0 : *zop->u.prox->exclusion),
2598                                          *zop->u.prox->relationType,
2599                                          *zop->u.prox->distance );
2600                 }
2601                 break;
2602             default:
2603                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2604                 return ZEBRA_FAIL;
2605             }
2606             *num_result_sets = 1;
2607             *result_sets = nmem_malloc(stream, *num_result_sets *
2608                                        sizeof(**result_sets));
2609             (*result_sets)[0] = rset;
2610         }
2611     }
2612     else if (zs->which == Z_RPNStructure_simple)
2613     {
2614         RSET rset;
2615         ZEBRA_RES res;
2616
2617         if (zs->u.simple->which == Z_Operand_APT)
2618         {
2619             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2620             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2621                                  attributeSet, stream, sort_sequence,
2622                                  num_bases, basenames, rset_nmem, &rset,
2623                                  kc);
2624             if (res != ZEBRA_OK)
2625                 return res;
2626         }
2627         else if (zs->u.simple->which == Z_Operand_resultSetId)
2628         {
2629             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2630             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2631             if (!rset)
2632             {
2633                 zebra_setError(zh,
2634                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2635                                zs->u.simple->u.resultSetId);
2636                 return ZEBRA_FAIL;
2637             }
2638             rset_dup(rset);
2639         }
2640         else
2641         {
2642             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2643             return ZEBRA_FAIL;
2644         }
2645         *num_result_sets = 1;
2646         *result_sets = nmem_malloc(stream, *num_result_sets *
2647                                    sizeof(**result_sets));
2648         (*result_sets)[0] = rset;
2649     }
2650     else
2651     {
2652         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2653         return ZEBRA_FAIL;
2654     }
2655     return ZEBRA_OK;
2656 }
2657
2658 struct scan_info_entry {
2659     char *term;
2660     ISAM_P isam_p;
2661 };
2662
2663 struct scan_info {
2664     struct scan_info_entry *list;
2665     ODR odr;
2666     int before, after;
2667     char prefix[20];
2668 };
2669
2670 static int scan_handle (char *name, const char *info, int pos, void *client)
2671 {
2672     int len_prefix, idx;
2673     struct scan_info *scan_info = (struct scan_info *) client;
2674
2675     len_prefix = strlen(scan_info->prefix);
2676     if (memcmp (name, scan_info->prefix, len_prefix))
2677         return 1;
2678     if (pos > 0)
2679         idx = scan_info->after - pos + scan_info->before;
2680     else
2681         idx = - pos - 1;
2682
2683     if (idx < 0)
2684         return 0;
2685     scan_info->list[idx].term = (char *)
2686         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2687     strcpy(scan_info->list[idx].term, name + len_prefix);
2688     assert (*info == sizeof(ISAM_P));
2689     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2690     return 0;
2691 }
2692
2693 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2694                               char **dst, const char *src)
2695 {
2696     char term_src[IT_MAX_WORD];
2697     char term_dst[IT_MAX_WORD];
2698
2699     zebra_term_untrans (zh, reg_type, term_src, src);
2700
2701     if (zh->iconv_from_utf8 != 0)
2702     {
2703         int len;
2704         char *inbuf = term_src;
2705         size_t inleft = strlen(term_src);
2706         char *outbuf = term_dst;
2707         size_t outleft = sizeof(term_dst)-1;
2708         size_t ret;
2709
2710         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2711                          &outbuf, &outleft);
2712         if (ret == (size_t)(-1))
2713             len = 0;
2714         else
2715             len = outbuf - term_dst;
2716         *dst = nmem_malloc(stream, len + 1);
2717         if (len > 0)
2718             memcpy (*dst, term_dst, len);
2719         (*dst)[len] = '\0';
2720     }
2721     else
2722         *dst = nmem_strdup(stream, term_src);
2723 }
2724
2725 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2726 {
2727     zint psysno = 0;
2728     struct it_key key;
2729     RSFD rfd;
2730
2731     yaz_log(YLOG_DEBUG, "count_set");
2732
2733     rset->hits_limit = zh->approx_limit;
2734
2735     *count = 0;
2736     rfd = rset_open(rset, RSETF_READ);
2737     while (rset_read(rfd, &key,0 /* never mind terms */))
2738     {
2739         if (key.mem[0] != psysno)
2740         {
2741             psysno = key.mem[0];
2742             if (rfd->counted_items >= rset->hits_limit)
2743                 break;
2744         }
2745     }
2746     rset_close (rfd);
2747     *count = rset->hits_count;
2748 }
2749
2750 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2751                    oid_value attributeset,
2752                    int num_bases, char **basenames,
2753                    int *position, int *num_entries, ZebraScanEntry **list,
2754                    int *is_partial, RSET limit_set, int return_zero)
2755 {
2756     int i;
2757     int pos = *position;
2758     int num = *num_entries;
2759     int before;
2760     int after;
2761     int base_no;
2762     char termz[IT_MAX_WORD+20];
2763     AttrType use;
2764     int use_value;
2765     const char *use_string = 0;
2766     struct scan_info *scan_info_array;
2767     ZebraScanEntry *glist;
2768     int ords[32], ord_no = 0;
2769     int ptr[32];
2770
2771     int bases_ok = 0;     /* no of databases with OK attribute */
2772     int errCode = 0;      /* err code (if any is not OK) */
2773     char *errString = 0;  /* addinfo */
2774
2775     unsigned reg_id;
2776     char *search_type = NULL;
2777     char rank_type[128];
2778     int complete_flag;
2779     int sort_flag;
2780     NMEM rset_nmem = NULL;
2781     struct rset_key_control *kc = 0;
2782
2783     *list = 0;
2784     *is_partial = 0;
2785
2786     if (attributeset == VAL_NONE)
2787         attributeset = VAL_BIB1;
2788
2789     if (!limit_set)
2790     {
2791         AttrType termset;
2792         int termset_value_numeric;
2793         const char *termset_value_string;
2794         attr_init(&termset, zapt, 8);
2795         termset_value_numeric =
2796             attr_find_ex(&termset, NULL, &termset_value_string);
2797         if (termset_value_numeric != -1)
2798         {
2799             char resname[32];
2800             const char *termset_name = 0;
2801
2802             if (termset_value_numeric != -2)
2803             {
2804
2805                 sprintf(resname, "%d", termset_value_numeric);
2806                 termset_name = resname;
2807             }
2808             else
2809                 termset_name = termset_value_string;
2810
2811             limit_set = resultSetRef (zh, termset_name);
2812         }
2813     }
2814
2815     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2816             pos, num, attributeset);
2817
2818     attr_init(&use, zapt, 1);
2819     use_value = attr_find_ex(&use, &attributeset, &use_string);
2820
2821     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2822                         rank_type, &complete_flag, &sort_flag))
2823     {
2824         *num_entries = 0;
2825         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2826         return ZEBRA_FAIL;
2827     }
2828     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2829
2830     if (use_value == -1)
2831         use_value = 1016;
2832     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2833     {
2834         data1_local_attribute *local_attr;
2835         attent attp;
2836         int ord;
2837
2838         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2839         {
2840             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2841                            basenames[base_no]);
2842             *num_entries = 0;
2843             return ZEBRA_FAIL;
2844         }
2845
2846         if (use_string &&
2847             (ord = zebraExplain_lookup_attr_str(zh->reg->zei, reg_id,
2848                                                 use_string)) >= 0)
2849         {
2850             /* we have a match for a raw string attribute */
2851             if (ord > 0)
2852                 ords[ord_no++] = ord;
2853             attp.local_attributes = 0;  /* no more attributes */
2854         }
2855         else
2856         {
2857             int r;
2858
2859             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2860                                       use_string)))
2861             {
2862                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2863                         attributeset, use_value);
2864                 if (r == -1)
2865                 {
2866                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2867                     if (use_string)
2868                         zebra_setError(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
2869                                        use_string);
2870                     else
2871                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
2872                                             use_value);
2873                 }
2874                 else
2875                 {
2876                     zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_SET, 0);
2877                 }
2878                 continue;
2879             }
2880         }
2881         bases_ok++;
2882         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2883              local_attr = local_attr->next)
2884         {
2885             ord = zebraExplain_lookup_attr_su(zh->reg->zei, reg_id,
2886                                               attp.attset_ordinal,
2887                                               local_attr->local);
2888             if (ord > 0)
2889                 ords[ord_no++] = ord;
2890         }
2891     }
2892     if (!bases_ok && errCode)
2893     {
2894         zebra_setError(zh, errCode, errString);
2895         *num_entries = 0;
2896         return ZEBRA_FAIL;
2897     }
2898     if (ord_no == 0)
2899     {
2900         *num_entries = 0;
2901         return ZEBRA_OK;
2902     }
2903     /* prepare dictionary scanning */
2904     if (num < 1)
2905     {
2906         *num_entries = 0;
2907         return ZEBRA_OK;
2908     }
2909     before = pos-1;
2910     if (before < 0)
2911         before = 0;
2912     after = 1+num-pos;
2913     if (after < 0)
2914         after = 0;
2915     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2916             "after=%d before+after=%d",
2917             pos, num, before, after, before+after);
2918     scan_info_array = (struct scan_info *)
2919         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2920     for (i = 0; i < ord_no; i++)
2921     {
2922         int j, prefix_len = 0;
2923         int before_tmp = before, after_tmp = after;
2924         struct scan_info *scan_info = scan_info_array + i;
2925         struct rpn_char_map_info rcmi;
2926
2927         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2928
2929         scan_info->before = before;
2930         scan_info->after = after;
2931         scan_info->odr = stream;
2932
2933         scan_info->list = (struct scan_info_entry *)
2934             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2935         for (j = 0; j<before+after; j++)
2936             scan_info->list[j].term = NULL;
2937
2938         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2939         termz[prefix_len] = 0;
2940         strcpy(scan_info->prefix, termz);
2941
2942         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2943             return ZEBRA_FAIL;
2944
2945         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2946                   scan_info, scan_handle);
2947     }
2948     glist = (ZebraScanEntry *)
2949         odr_malloc(stream, (before+after)*sizeof(*glist));
2950
2951     rset_nmem = nmem_create();
2952     kc = zebra_key_control_create(zh);
2953
2954     /* consider terms after main term */
2955     for (i = 0; i < ord_no; i++)
2956         ptr[i] = before;
2957
2958     *is_partial = 0;
2959     for (i = 0; i<after; i++)
2960     {
2961         int j, j0 = -1;
2962         const char *mterm = NULL;
2963         const char *tst;
2964         RSET rset = 0;
2965         int lo = i + pos-1; /* offset in result list */
2966
2967         /* find: j0 is the first of the minimal values */
2968         for (j = 0; j < ord_no; j++)
2969         {
2970             if (ptr[j] < before+after && ptr[j] >= 0 &&
2971                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2972                 (!mterm || strcmp (tst, mterm) < 0))
2973             {
2974                 j0 = j;
2975                 mterm = tst;
2976             }
2977         }
2978         if (j0 == -1)
2979             break;  /* no value found, stop */
2980
2981         /* get result set for first one , but only if it's within bounds */
2982         if (lo >= 0)
2983         {
2984             /* get result set for first term */
2985             zebra_term_untrans_iconv(zh, stream->mem, reg_id,
2986                                      &glist[lo].term, mterm);
2987             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2988                               glist[lo].term, strlen(glist[lo].term),
2989                               NULL, 0, zapt->term->which, rset_nmem,
2990                               kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
2991                               0 /* term_ref_id_str */);
2992         }
2993         ptr[j0]++; /* move index for this set .. */
2994         /* get result set for remaining scan terms */
2995         for (j = j0+1; j<ord_no; j++)
2996         {
2997             if (ptr[j] < before+after && ptr[j] >= 0 &&
2998                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2999                 !strcmp (tst, mterm))
3000             {
3001                 if (lo >= 0)
3002                 {
3003                     RSET rsets[2];
3004
3005                     rsets[0] = rset;
3006                     rsets[1] =
3007                         rset_trunc(
3008                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
3009                             glist[lo].term,
3010                             strlen(glist[lo].term), NULL, 0,
3011                             zapt->term->which,rset_nmem,
3012                             kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3013                             0 /* term_ref_id_str */ );
3014                     rset = rsmulti_or_create(rset_nmem, kc,
3015                                              kc->scope, 0 /* termid */,
3016                                              2, rsets);
3017                 }
3018                 ptr[j]++;
3019             }
3020         }
3021         if (lo >= 0)
3022         {
3023             zint count;
3024             /* merge with limit_set if given */
3025             if (limit_set)
3026             {
3027                 RSET rsets[2];
3028                 rsets[0] = rset;
3029                 rsets[1] = rset_dup(limit_set);
3030
3031                 rset = rsmulti_and_create(rset_nmem, kc,
3032                                           kc->scope,
3033                                           2, rsets);
3034             }
3035             /* count it */
3036             count_set(zh, rset, &count);
3037             glist[lo].occurrences = count;
3038             rset_delete(rset);
3039         }
3040     }
3041     if (i < after)
3042     {
3043         *num_entries -= (after-i);
3044         *is_partial = 1;
3045         if (*num_entries < 0)
3046         {
3047             (*kc->dec)(kc);
3048             nmem_destroy(rset_nmem);
3049             *num_entries = 0;
3050             return ZEBRA_OK;
3051         }
3052     }
3053     /* consider terms before main term */
3054     for (i = 0; i<ord_no; i++)
3055         ptr[i] = 0;
3056
3057     for (i = 0; i<before; i++)
3058     {
3059         int j, j0 = -1;
3060         const char *mterm = NULL;
3061         const char *tst;
3062         RSET rset;
3063         int lo = before-1-i; /* offset in result list */
3064         zint count;
3065
3066         for (j = 0; j <ord_no; j++)
3067         {
3068             if (ptr[j] < before && ptr[j] >= 0 &&
3069                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3070                 (!mterm || strcmp (tst, mterm) > 0))
3071             {
3072                 j0 = j;
3073                     mterm = tst;
3074             }
3075         }
3076         if (j0 == -1)
3077             break;
3078
3079         zebra_term_untrans_iconv(zh, stream->mem, reg_id,
3080                                  &glist[lo].term, mterm);
3081
3082         rset = rset_trunc
3083             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
3084              glist[lo].term, strlen(glist[lo].term),
3085              NULL, 0, zapt->term->which, rset_nmem,
3086              kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3087              0 /* term_ref_id_str */);
3088
3089         ptr[j0]++;
3090
3091         for (j = j0+1; j<ord_no; j++)
3092         {
3093             if (ptr[j] < before && ptr[j] >= 0 &&
3094                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3095                 !strcmp (tst, mterm))
3096             {
3097                 RSET rsets[2];
3098
3099                 rsets[0] = rset;
3100                 rsets[1] = rset_trunc(
3101                     zh,
3102                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
3103                     glist[lo].term,
3104                     strlen(glist[lo].term), NULL, 0,
3105                     zapt->term->which, rset_nmem,
3106                     kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3107                     0 /* term_ref_id_str */);
3108                 rset = rsmulti_or_create(rset_nmem, kc,
3109                                          kc->scope, 0 /* termid */, 2, rsets);
3110
3111                 ptr[j]++;
3112             }
3113         }
3114         if (limit_set)
3115         {
3116             RSET rsets[2];
3117             rsets[0] = rset;
3118             rsets[1] = rset_dup(limit_set);
3119
3120             rset = rsmulti_and_create(rset_nmem, kc,
3121                                       kc->scope, 2, rsets);
3122         }
3123         count_set(zh, rset, &count);
3124         glist[lo].occurrences = count;
3125         rset_delete (rset);
3126     }
3127     (*kc->dec)(kc);
3128     nmem_destroy(rset_nmem);
3129     i = before-i;
3130     if (i)
3131     {
3132         *is_partial = 1;
3133         *position -= i;
3134         *num_entries -= i;
3135         if (*num_entries <= 0)
3136         {
3137             *num_entries = 0;
3138             return ZEBRA_OK;
3139         }
3140     }
3141
3142     *list = glist + i;               /* list is set to first 'real' entry */
3143
3144     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3145             *position, *num_entries);
3146     return ZEBRA_OK;
3147 }
3148
3149 /*
3150  * Local variables:
3151  * c-basic-offset: 4
3152  * indent-tabs-mode: nil
3153  * End:
3154  * vim: shiftwidth=4 tabstop=8 expandtab
3155  */
3156