index/extract.c

   1 /* $Id: extract.c,v 1.259 2007-08-21 11:06:47 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #endif
  29 #if HAVE_UNISTD_H
  30 #include <unistd.h>
  31 #endif
  32 #include <fcntl.h>
  33
  34 #include "index.h"
  35 #include "orddict.h"
  36 #include <direntz.h>
  37 #include <charmap.h>
  38
  39 static int log_level_extract = 0;
  40 static int log_level_details = 0;
  41 static int log_level_initialized = 0;
  42
  43 /* 1 if we use eliminitate identical delete/insert keys */
  44 /* eventually this the 0-case code will be removed */
  45 #define FLUSH2 1
  46
  47 void extract_flush_record_keys2(ZebraHandle zh, zint sysno,
  48                                 zebra_rec_keys_t ins_keys,
  49                                 zint ins_rank,
  50                                 zebra_rec_keys_t del_keys,
  51                                 zint del_rank);
  52
  53 static void zebra_init_log_level(void)
  54 {
  55     if (!log_level_initialized)
  56     {
  57         log_level_initialized = 1;
  58
  59         log_level_extract = yaz_log_module_level("extract");
  60         log_level_details = yaz_log_module_level("indexdetails");
  61     }
  62 }
  63
  64 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
  65                                       int cmd, zebra_rec_keys_t reckeys,
  66                                       zint staticrank);
  67 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
  68                                     int cmd, zebra_rec_keys_t skp);
  69 static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid);
  70 static void extract_token_add (RecWord *p);
  71
  72 static void check_log_limit(ZebraHandle zh)
  73 {
  74     if (zh->records_processed + zh->records_skipped == zh->m_file_verbose_limit)
  75     {
  76         yaz_log(YLOG_LOG, "More than %d file log entries. Omitting rest",
  77                 zh->m_file_verbose_limit);
  78     }
  79 }
  80
  81 static void logRecord (ZebraHandle zh)
  82 {
  83     check_log_limit(zh);
  84     ++zh->records_processed;
  85     if (!(zh->records_processed % 1000))
  86     {
  87         yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
  88                 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
  89                 zh->records_processed, zh->records_inserted,
  90                 zh->records_updated, zh->records_deleted);
  91     }
  92 }
  93
  94 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
  95 {
  96     int i;
  97     for (i = 0; i<256; i++)
  98     {
  99         if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
 100             ctrl->seqno[i] = 1;
 101         else
 102             ctrl->seqno[i] = 0;
 103     }
 104     ctrl->flagShowRecords = !zh->m_flag_rw;
 105 }
 106
 107
 108 static void extract_add_index_string (RecWord *p,
 109                                       zinfo_index_category_t cat,
 110                                       const char *str, int length);
 111
 112 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
 113
 114 static void extract_init(struct recExtractCtrl *p, RecWord *w)
 115 {
 116     w->seqno = 1;
 117     w->index_name = "any";
 118     w->index_type = 'w';
 119     w->extractCtrl = p;
 120     w->record_id = 0;
 121     w->section_id = 0;
 122     w->segment = 0;
 123 }
 124
 125 struct snip_rec_info {
 126     ZebraHandle zh;
 127     zebra_snippets *snippets;
 128 };
 129
 130
 131 static void snippet_add_complete_field(RecWord *p)
 132 {
 133
 134 }
 135
 136 static void snippet_add_incomplete_field(RecWord *p, int ord)
 137 {
 138     struct snip_rec_info *h = p->extractCtrl->handle;
 139     ZebraHandle zh = h->zh;
 140     const char *b = p->term_buf;
 141     int remain = p->term_len;
 142     int first = 1;
 143     const char **map = 0;
 144     const char *start = b;
 145
 146     if (remain > 0)
 147         map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
 148
 149     while (map)
 150     {
 151         char buf[IT_MAX_WORD+1];
 152         const char *last = b;
 153         int i, remain;
 154
 155         /* Skip spaces */
 156         while (map && *map && **map == *CHR_SPACE)
 157         {
 158             remain = p->term_len - (b - p->term_buf);
 159             last = b;
 160             if (remain > 0)
 161                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b,
 162                                        remain, 0);
 163             else
 164                 map = 0;
 165         }
 166         if (!map)
 167             break;
 168         if (start != last)
 169         {
 170             zebra_snippets_appendn(h->snippets, p->seqno, 1, ord,
 171                                    start, last - start);
 172
 173         }
 174         start = last;
 175
 176         i = 0;
 177         while (map && *map && **map != *CHR_SPACE)
 178         {
 179             const char *cp = *map;
 180
 181             while (i < IT_MAX_WORD && *cp)
 182                 buf[i++] = *(cp++);
 183             remain = p->term_len - (b - p->term_buf);
 184             last = b;
 185             if (remain > 0)
 186                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
 187             else
 188                 map = 0;
 189         }
 190         if (!i)
 191             return;
 192
 193         if (first)
 194         {
 195             first = 0;
 196             if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type))
 197             {
 198                 /* first in field marker */
 199                 p->seqno++;
 200             }
 201         }
 202         if (start != last)
 203             zebra_snippets_appendn(h->snippets, p->seqno, 0, ord,
 204                                    start, last - start);
 205         start = last;
 206         p->seqno++;
 207     }
 208
 209 }
 210
 211 static void snippet_token_add(RecWord *p)
 212 {
 213     struct snip_rec_info *h = p->extractCtrl->handle;
 214     ZebraHandle zh = h->zh;
 215
 216     if (zebra_maps_is_index(zh->reg->zebra_maps, p->index_type))
 217     {
 218         ZebraExplainInfo zei = zh->reg->zei;
 219         int ch = zebraExplain_lookup_attr_str(
 220             zei, zinfo_index_category_index, p->index_type, p->index_name);
 221
 222         if (zebra_maps_is_complete (h->zh->reg->zebra_maps, p->index_type))
 223             snippet_add_complete_field (p);
 224         else
 225             snippet_add_incomplete_field(p, ch);
 226     }
 227 }
 228
 229 static void snippet_schema_add(
 230     struct recExtractCtrl *p, Odr_oid *oid)
 231 {
 232
 233 }
 234
 235 void extract_snippet(ZebraHandle zh, zebra_snippets *sn,
 236                      struct ZebraRecStream *stream,
 237                      RecType rt, void *recTypeClientData)
 238 {
 239     struct recExtractCtrl extractCtrl;
 240     struct snip_rec_info info;
 241     int r;
 242
 243     extractCtrl.stream = stream;
 244     extractCtrl.first_record = 1;
 245     extractCtrl.init = extract_init;
 246     extractCtrl.tokenAdd = snippet_token_add;
 247     extractCtrl.schemaAdd = snippet_schema_add;
 248     assert(zh->reg);
 249     assert(zh->reg->dh);
 250
 251     extractCtrl.dh = zh->reg->dh;
 252
 253     info.zh = zh;
 254     info.snippets = sn;
 255     extractCtrl.handle = &info;
 256     extractCtrl.match_criteria[0] = '\0';
 257     extractCtrl.staticrank = 0;
 258     extractCtrl.action = action_insert;
 259
 260     init_extractCtrl(zh, &extractCtrl);
 261
 262     r = (*rt->extract)(recTypeClientData, &extractCtrl);
 263
 264 }
 265
 266 static void searchRecordKey(ZebraHandle zh,
 267                             zebra_rec_keys_t reckeys,
 268                             const char *index_name,
 269                             const char **ws, int ws_length)
 270 {
 271     int i;
 272     int ch = -1;
 273     zinfo_index_category_t cat = zinfo_index_category_index;
 274
 275     for (i = 0; i<ws_length; i++)
 276         ws[i] = NULL;
 277
 278     if (ch < 0)
 279         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, '0', index_name);
 280     if (ch < 0)
 281         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'p', index_name);
 282     if (ch < 0)
 283         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'w', index_name);
 284
 285     if (ch < 0)
 286         return ;
 287
 288     if (zebra_rec_keys_rewind(reckeys))
 289     {
 290         zint startSeq = -1;
 291         const char *str;
 292         size_t slen;
 293         struct it_key key;
 294         zint seqno;
 295         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
 296         {
 297             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
 298
 299             seqno = key.mem[key.len-1];
 300
 301             if (key.mem[0] == ch)
 302             {
 303                 zint woff;
 304
 305                 if (startSeq == -1)
 306                     startSeq = seqno;
 307                 woff = seqno - startSeq;
 308                 if (woff >= 0 && woff < ws_length)
 309                     ws[woff] = str;
 310             }
 311         }
 312     }
 313 }
 314
 315 #define FILE_MATCH_BLANK "\t "
 316
 317 static char *get_match_from_spec(ZebraHandle zh,
 318                           zebra_rec_keys_t reckeys,
 319                           const char *fname, const char *spec)
 320 {
 321     static char dstBuf[2048];      /* static here ??? */
 322     char *dst = dstBuf;
 323     const char *s = spec;
 324
 325     while (1)
 326     {
 327         for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
 328             ;
 329         if (!*s)
 330             break;
 331         if (*s == '(')
 332         {
 333             const char *ws[32];
 334             char attset_str[64], attname_str[64];
 335             int i;
 336             int first = 1;
 337
 338             for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 339                 ;
 340             for (i = 0; *s && *s != ',' && *s != ')' &&
 341                      !strchr(FILE_MATCH_BLANK, *s); s++)
 342                 if (i+1 < sizeof(attset_str))
 343                     attset_str[i++] = *s;
 344             attset_str[i] = '\0';
 345
 346             for (; strchr(FILE_MATCH_BLANK, *s); s++)
 347                 ;
 348             if (*s != ',')
 349                 strcpy(attname_str, attset_str);
 350             else
 351             {
 352                 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 353                     ;
 354                 for (i = 0; *s && *s != ')' &&
 355                          !strchr(FILE_MATCH_BLANK, *s); s++)
 356                     if (i+1 < sizeof(attname_str))
 357                         attname_str[i++] = *s;
 358                 attname_str[i] = '\0';
 359             }
 360
 361             searchRecordKey (zh, reckeys, attname_str, ws, 32);
 362
 363             if (*s != ')')
 364             {
 365                 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
 366                       spec, zh->m_group ? zh->m_group : "none");
 367                 return NULL;
 368             }
 369             s++;
 370
 371             for (i = 0; i<32; i++)
 372                 if (ws[i])
 373                 {
 374                     if (first)
 375                     {
 376                         *dst++ = ' ';
 377                         first = 0;
 378                     }
 379                     strcpy (dst, ws[i]);
 380                     dst += strlen(ws[i]);
 381                 }
 382             if (first)
 383             {
 384                 yaz_log (YLOG_WARN, "Record didn't contain match"
 385                       " fields in (%s,%s)", attset_str, attname_str);
 386                 return NULL;
 387             }
 388         }
 389         else if (*s == '$')
 390         {
 391             int spec_len;
 392             char special[64];
 393             const char *spec_src = NULL;
 394             const char *s1 = ++s;
 395             while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
 396                 s1++;
 397
 398             spec_len = s1 - s;
 399             if (spec_len > sizeof(special)-1)
 400                 spec_len = sizeof(special)-1;
 401             memcpy (special, s, spec_len);
 402             special[spec_len] = '\0';
 403             s = s1;
 404
 405             if (!strcmp (special, "group"))
 406                 spec_src = zh->m_group;
 407             else if (!strcmp (special, "database"))
 408                 spec_src = zh->basenames[0];
 409             else if (!strcmp (special, "filename")) {
 410                 spec_src = fname;
 411             }
 412             else if (!strcmp (special, "type"))
 413                 spec_src = zh->m_record_type;
 414             else
 415                 spec_src = NULL;
 416             if (spec_src)
 417             {
 418                 strcpy (dst, spec_src);
 419                 dst += strlen (spec_src);
 420             }
 421         }
 422         else if (*s == '\"' || *s == '\'')
 423         {
 424             int stopMarker = *s++;
 425             char tmpString[64];
 426             int i = 0;
 427
 428             while (*s && *s != stopMarker)
 429             {
 430                 if (i+1 < sizeof(tmpString))
 431                     tmpString[i++] = *s++;
 432             }
 433             if (*s)
 434                 s++;
 435             tmpString[i] = '\0';
 436             strcpy (dst, tmpString);
 437             dst += strlen (tmpString);
 438         }
 439         else
 440         {
 441             yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
 442                   spec, zh->m_group ? zh->m_group : "none");
 443             return NULL;
 444         }
 445         *dst++ = 1;
 446     }
 447     if (dst == dstBuf)
 448     {
 449         yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
 450               fname, zh->m_group ? zh->m_group : "none");
 451         return NULL;
 452     }
 453     *dst = '\0';
 454     return dstBuf;
 455 }
 456
 457 struct recordLogInfo {
 458     const char *fname;
 459     int recordOffset;
 460     struct recordGroup *rGroup;
 461 };
 462
 463 static void all_matches_add(struct recExtractCtrl *ctrl)
 464 {
 465     RecWord word;
 466     extract_init(ctrl, &word);
 467     word.index_name = "_ALLRECORDS";
 468     word.index_type = 'w';
 469     word.seqno = 1;
 470     extract_add_index_string (&word, zinfo_index_category_alwaysmatches,
 471                               "", 0);
 472 }
 473
 474 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
 475                                        struct ZebraRecStream *stream,
 476                                        enum zebra_recctrl_action_t action,
 477                                        int test_mode,
 478                                        const char *recordType,
 479                                        zint *sysno,
 480                                        const char *match_criteria,
 481                                        const char *fname,
 482                                        RecType recType,
 483                                        void *recTypeClientData);
 484
 485
 486 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
 487                              int deleteFlag)
 488 {
 489     ZEBRA_RES r = ZEBRA_OK;
 490     int i, fd;
 491     char gprefix[128];
 492     char ext[128];
 493     char ext_res[128];
 494     struct file_read_info *fi = 0;
 495     const char *original_record_type = 0;
 496     RecType recType;
 497     void *recTypeClientData;
 498     struct ZebraRecStream stream, *streamp;
 499
 500     zebra_init_log_level();
 501
 502     if (!zh->m_group || !*zh->m_group)
 503         *gprefix = '\0';
 504     else
 505         sprintf (gprefix, "%s.", zh->m_group);
 506
 507     yaz_log(log_level_extract, "zebra_extract_file %s", fname);
 508
 509     /* determine file extension */
 510     *ext = '\0';
 511     for (i = strlen(fname); --i >= 0; )
 512         if (fname[i] == '/')
 513             break;
 514         else if (fname[i] == '.')
 515         {
 516             strcpy (ext, fname+i+1);
 517             break;
 518         }
 519     /* determine file type - depending on extension */
 520     original_record_type = zh->m_record_type;
 521     if (!zh->m_record_type)
 522     {
 523         sprintf (ext_res, "%srecordType.%s", gprefix, ext);
 524         zh->m_record_type = res_get (zh->res, ext_res);
 525     }
 526     if (!zh->m_record_type)
 527     {
 528         check_log_limit(zh);
 529         if (zh->records_processed + zh->records_skipped
 530             < zh->m_file_verbose_limit)
 531             yaz_log (YLOG_LOG, "? %s", fname);
 532         zh->records_skipped++;
 533         return 0;
 534     }
 535     /* determine match criteria */
 536     if (!zh->m_record_id)
 537     {
 538         sprintf (ext_res, "%srecordId.%s", gprefix, ext);
 539         zh->m_record_id = res_get (zh->res, ext_res);
 540     }
 541
 542     if (!(recType =
 543           recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
 544                           &recTypeClientData)))
 545     {
 546         yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
 547         return ZEBRA_FAIL;
 548     }
 549
 550     switch(recType->version)
 551     {
 552     case 0:
 553         break;
 554     default:
 555         yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
 556     }
 557     if (sysno && deleteFlag)
 558     {
 559         streamp = 0;
 560         fi = 0;
 561     }
 562     else
 563     {
 564         char full_rep[1024];
 565
 566         if (zh->path_reg && !yaz_is_abspath (fname))
 567         {
 568             strcpy (full_rep, zh->path_reg);
 569             strcat (full_rep, "/");
 570             strcat (full_rep, fname);
 571         }
 572         else
 573             strcpy (full_rep, fname);
 574
 575         if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
 576         {
 577             yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
 578             zh->m_record_type = original_record_type;
 579             return ZEBRA_FAIL;
 580         }
 581         streamp = &stream;
 582         zebra_create_stream_fd(streamp, fd, 0);
 583     }
 584     r = zebra_extract_records_stream(zh, streamp,
 585                                      deleteFlag ?
 586                                      action_delete : action_update,
 587                                      0, /* tst_mode */
 588                                      zh->m_record_type,
 589                                      sysno,
 590                                      0, /*match_criteria */
 591                                      fname,
 592                                      recType, recTypeClientData);
 593     if (streamp)
 594         stream.destroy(streamp);
 595     zh->m_record_type = original_record_type;
 596     return r;
 597 }
 598
 599 /*
 600   If sysno is provided, then it's used to identify the reocord.
 601   If not, and match_criteria is provided, then sysno is guessed
 602   If not, and a record is provided, then sysno is got from there
 603
 604  */
 605
 606 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
 607                                       const char *buf, size_t buf_size,
 608                                       enum zebra_recctrl_action_t action,
 609                                       int test_mode,
 610                                       const char *recordType,
 611                                       zint *sysno,
 612                                       const char *match_criteria,
 613                                       const char *fname)
 614 {
 615     struct ZebraRecStream stream;
 616     ZEBRA_RES res;
 617     void *clientData;
 618     RecType recType = 0;
 619
 620     if (recordType && *recordType)
 621     {
 622         yaz_log(log_level_extract,
 623                 "Record type explicitly specified: %s", recordType);
 624         recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
 625                                   &clientData);
 626     }
 627     else
 628     {
 629         if (!(zh->m_record_type))
 630         {
 631             yaz_log (YLOG_WARN, "No such record type defined");
 632             return ZEBRA_FAIL;
 633         }
 634         yaz_log(log_level_extract, "Get record type from rgroup: %s",
 635                 zh->m_record_type);
 636         recType = recType_byName (zh->reg->recTypes, zh->res,
 637                                   zh->m_record_type, &clientData);
 638         recordType = zh->m_record_type;
 639     }
 640
 641     if (!recType)
 642     {
 643         yaz_log (YLOG_WARN, "No such record type: %s", recordType);
 644         return ZEBRA_FAIL;
 645     }
 646
 647     zebra_create_stream_mem(&stream, buf, buf_size);
 648
 649     res = zebra_extract_records_stream(zh, &stream,
 650                                        action,
 651                                        test_mode,
 652                                        recordType,
 653                                        sysno,
 654                                        match_criteria,
 655                                        fname,
 656                                        recType, clientData);
 657     stream.destroy(&stream);
 658     return res;
 659 }
 660
 661 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
 662                                        struct ZebraRecStream *stream,
 663                                        enum zebra_recctrl_action_t action,
 664                                        int test_mode,
 665                                        const char *recordType,
 666                                        zint *sysno,
 667                                        const char *match_criteria,
 668                                        const char *fname,
 669                                        RecType recType,
 670                                        void *recTypeClientData)
 671 {
 672     ZEBRA_RES res = ZEBRA_OK;
 673     while (1)
 674     {
 675         int more = 0;
 676         res = zebra_extract_record_stream(zh, stream,
 677                                           action,
 678                                           test_mode,
 679                                           recordType,
 680                                           sysno,
 681                                           match_criteria,
 682                                           fname,
 683                                           recType, recTypeClientData, &more);
 684         if (!more)
 685         {
 686             res = ZEBRA_OK;
 687             break;
 688         }
 689         if (res != ZEBRA_OK)
 690             break;
 691         if (sysno)
 692             break;
 693     }
 694     return res;
 695 }
 696
 697
 698 static WRBUF wrbuf_hex_str(const char *cstr)
 699 {
 700     size_t i;
 701     WRBUF w = wrbuf_alloc();
 702     for (i = 0; cstr[i]; i++)
 703     {
 704         if (cstr[i] < ' ' || cstr[i] > 126)
 705             wrbuf_printf(w, "\\%02X", cstr[i] & 0xff);
 706         else
 707             wrbuf_putc(w, cstr[i]);
 708     }
 709     return w;
 710 }
 711
 712 ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
 713                                       struct ZebraRecStream *stream,
 714                                       enum zebra_recctrl_action_t action,
 715                                       int test_mode,
 716                                       const char *recordType,
 717                                       zint *sysno,
 718                                       const char *match_criteria,
 719                                       const char *fname,
 720                                       RecType recType,
 721                                       void *recTypeClientData,
 722                                       int *more)
 723
 724 {
 725     zint sysno0 = 0;
 726     RecordAttr *recordAttr;
 727     struct recExtractCtrl extractCtrl;
 728     int r;
 729     const char *matchStr = 0;
 730     Record rec;
 731     off_t start_offset = 0, end_offset = 0;
 732     const char *pr_fname = fname;  /* filename to print .. */
 733     int show_progress = zh->records_processed + zh->records_skipped
 734         < zh->m_file_verbose_limit ? 1:0;
 735
 736     zebra_init_log_level();
 737
 738     if (!pr_fname)
 739         pr_fname = "<no file>";  /* make it printable if file is omitted */
 740
 741     zebra_rec_keys_reset(zh->reg->keys);
 742     zebra_rec_keys_reset(zh->reg->sortKeys);
 743
 744     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
 745     {
 746         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
 747                                       zh->m_explain_database))
 748             return ZEBRA_FAIL;
 749     }
 750
 751     if (stream)
 752     {
 753         off_t null_offset = 0;
 754         extractCtrl.stream = stream;
 755
 756         start_offset = stream->tellf(stream);
 757
 758         extractCtrl.first_record = start_offset ? 0 : 1;
 759
 760         stream->endf(stream, &null_offset);;
 761
 762         extractCtrl.init = extract_init;
 763         extractCtrl.tokenAdd = extract_token_add;
 764         extractCtrl.schemaAdd = extract_schema_add;
 765         extractCtrl.dh = zh->reg->dh;
 766         extractCtrl.handle = zh;
 767         extractCtrl.match_criteria[0] = '\0';
 768         extractCtrl.staticrank = 0;
 769         extractCtrl.action = action;
 770
 771         init_extractCtrl(zh, &extractCtrl);
 772
 773         extract_set_store_data_prepare(&extractCtrl);
 774
 775         r = (*recType->extract)(recTypeClientData, &extractCtrl);
 776
 777         if (action == action_update)
 778         {
 779             action = extractCtrl.action;
 780         }
 781
 782         switch (r)
 783         {
 784         case RECCTRL_EXTRACT_EOF:
 785             return ZEBRA_FAIL;
 786         case RECCTRL_EXTRACT_ERROR_GENERIC:
 787             /* error occured during extraction ... */
 788             yaz_log (YLOG_WARN, "extract error: generic");
 789             return ZEBRA_FAIL;
 790         case RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER:
 791             /* error occured during extraction ... */
 792             yaz_log (YLOG_WARN, "extract error: no such filter");
 793             return ZEBRA_FAIL;
 794         case RECCTRL_EXTRACT_SKIP:
 795             if (show_progress)
 796                 yaz_log (YLOG_LOG, "skip %s %s " ZINT_FORMAT,
 797                          recordType, pr_fname, (zint) start_offset);
 798             *more = 1;
 799
 800             end_offset = stream->endf(stream, 0);
 801             if (end_offset)
 802                 stream->seekf(stream, end_offset);
 803
 804             return ZEBRA_OK;
 805         case RECCTRL_EXTRACT_OK:
 806             break;
 807         default:
 808             yaz_log (YLOG_WARN, "extract error: unknown error: %d", r);
 809             return ZEBRA_FAIL;
 810         }
 811         end_offset = stream->endf(stream, 0);
 812         if (end_offset)
 813             stream->seekf(stream, end_offset);
 814         else
 815             end_offset = stream->tellf(stream);
 816
 817         all_matches_add(&extractCtrl);
 818
 819         if (extractCtrl.match_criteria[0])
 820             match_criteria = extractCtrl.match_criteria;
 821     }
 822
 823     *more = 1;
 824     if (!sysno)
 825     {
 826         sysno = &sysno0;
 827
 828         if (match_criteria && *match_criteria) {
 829             matchStr = match_criteria;
 830         } else {
 831             if (zh->m_record_id && *zh->m_record_id) {
 832                 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
 833                                                zh->m_record_id);
 834                 if (!matchStr)
 835                 {
 836                     yaz_log (YLOG_LOG, "error %s %s " ZINT_FORMAT, recordType,
 837                              pr_fname, (zint) start_offset);
 838                     return ZEBRA_FAIL;
 839                 }
 840             }
 841         }
 842         if (matchStr)
 843         {
 844             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 845             char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
 846                                           matchStr);
 847
 848
 849             if (log_level_extract)
 850             {
 851                 WRBUF w = wrbuf_hex_str(matchStr);
 852                 yaz_log(log_level_extract, "matchStr: %s", wrbuf_cstr(w));
 853                 wrbuf_destroy(w);
 854             }
 855             if (rinfo)
 856             {
 857                 assert(*rinfo == sizeof(*sysno));
 858                 memcpy (sysno, rinfo+1, sizeof(*sysno));
 859             }
 860        }
 861     }
 862     if (zebra_rec_keys_empty(zh->reg->keys))
 863     {
 864         /* the extraction process returned no information - the record
 865            is probably empty - unless flagShowRecords is in use */
 866         if (test_mode)
 867             return ZEBRA_OK;
 868     }
 869
 870     if (! *sysno)
 871     {
 872         /* new record */
 873         if (action == action_delete)
 874         {
 875             yaz_log (YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 876                          pr_fname, (zint) start_offset);
 877             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
 878             return ZEBRA_FAIL;
 879         }
 880         else if (action == action_replace)
 881         {
 882             yaz_log (YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 883                          pr_fname, (zint) start_offset);
 884             yaz_log (YLOG_WARN, "cannot update record above (seems new)");
 885             return ZEBRA_FAIL;
 886         }
 887         if (show_progress)
 888             yaz_log (YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
 889                      (zint) start_offset);
 890         rec = rec_new (zh->reg->records);
 891
 892         *sysno = rec->sysno;
 893
 894         recordAttr = rec_init_attr (zh->reg->zei, rec);
 895         if (extractCtrl.staticrank < 0)
 896         {
 897             yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0");
 898             extractCtrl.staticrank = 0;
 899         }
 900
 901         if (matchStr)
 902         {
 903             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 904             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
 905                             sizeof(*sysno), sysno);
 906         }
 907
 908         extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 909 #if FLUSH2
 910         extract_flush_record_keys2(zh, *sysno,
 911                                    zh->reg->keys, extractCtrl.staticrank,
 912                                    0, recordAttr->staticrank);
 913 #else
 914         extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 915                                   extractCtrl.staticrank);
 916 #endif
 917         recordAttr->staticrank = extractCtrl.staticrank;
 918         zh->records_inserted++;
 919     }
 920     else
 921     {
 922         /* record already exists */
 923         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 924         zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
 925         if (action == action_insert)
 926         {
 927             yaz_log (YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
 928                          recordType, pr_fname, (zint) start_offset);
 929             logRecord(zh);
 930             return ZEBRA_FAIL;
 931         }
 932
 933         rec = rec_get (zh->reg->records, *sysno);
 934         assert (rec);
 935
 936         recordAttr = rec_init_attr (zh->reg->zei, rec);
 937
 938         /* decrease total size */
 939         zebraExplain_recordBytesIncrement (zh->reg->zei,
 940                                            - recordAttr->recordSize);
 941
 942         zebra_rec_keys_set_buf(delkeys,
 943                                rec->info[recInfo_delKeys],
 944                                rec->size[recInfo_delKeys],
 945                                0);
 946         zebra_rec_keys_set_buf(sortKeys,
 947                                rec->info[recInfo_sortKeys],
 948                                rec->size[recInfo_sortKeys],
 949                                0);
 950
 951         extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
 952 #if !FLUSH2
 953         extract_flush_record_keys(zh, *sysno, 0, delkeys,
 954                                   recordAttr->staticrank);
 955 #endif
 956         if (action == action_delete)
 957         {
 958             /* record going to be deleted */
 959 #if FLUSH2
 960             extract_flush_record_keys2(zh, *sysno, 0, recordAttr->staticrank,
 961                                        delkeys, recordAttr->staticrank);
 962 #endif
 963             if (zebra_rec_keys_empty(delkeys))
 964             {
 965                 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 966                         pr_fname, (zint) start_offset);
 967                 yaz_log(YLOG_WARN, "cannot delete file above, "
 968                         "storeKeys false (3)");
 969             }
 970             else
 971             {
 972                 if (show_progress)
 973                     yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 974                             pr_fname, (zint) start_offset);
 975                 zh->records_deleted++;
 976                 if (matchStr)
 977                 {
 978                     int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 979                     dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
 980                 }
 981                 rec_del (zh->reg->records, &rec);
 982             }
 983             zebra_rec_keys_close(delkeys);
 984             zebra_rec_keys_close(sortKeys);
 985             rec_free(&rec);
 986             logRecord(zh);
 987             return ZEBRA_OK;
 988         }
 989         else
 990         {   /* update or special_update */
 991             if (show_progress)
 992                 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 993                         pr_fname, (zint) start_offset);
 994             extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 995
 996 #if FLUSH2
 997             extract_flush_record_keys2(zh, *sysno,
 998                                        zh->reg->keys, extractCtrl.staticrank,
 999                                        delkeys, recordAttr->staticrank);
1000 #else
1001             extract_flush_record_keys(zh, *sysno, 1,
1002                                       zh->reg->keys, extractCtrl.staticrank);
1003 #endif
1004             recordAttr->staticrank = extractCtrl.staticrank;
1005             zh->records_updated++;
1006         }
1007         zebra_rec_keys_close(delkeys);
1008         zebra_rec_keys_close(sortKeys);
1009     }
1010     /* update file type */
1011     xfree (rec->info[recInfo_fileType]);
1012     rec->info[recInfo_fileType] =
1013         rec_strdup (recordType, &rec->size[recInfo_fileType]);
1014
1015     /* update filename */
1016     xfree (rec->info[recInfo_filename]);
1017     rec->info[recInfo_filename] =
1018         rec_strdup (fname, &rec->size[recInfo_filename]);
1019
1020     /* update delete keys */
1021     xfree (rec->info[recInfo_delKeys]);
1022     if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
1023     {
1024         zebra_rec_keys_get_buf(zh->reg->keys,
1025                                &rec->info[recInfo_delKeys],
1026                                &rec->size[recInfo_delKeys]);
1027     }
1028     else
1029     {
1030         rec->info[recInfo_delKeys] = NULL;
1031         rec->size[recInfo_delKeys] = 0;
1032     }
1033     /* update sort keys */
1034     xfree (rec->info[recInfo_sortKeys]);
1035
1036     zebra_rec_keys_get_buf(zh->reg->sortKeys,
1037                            &rec->info[recInfo_sortKeys],
1038                            &rec->size[recInfo_sortKeys]);
1039
1040     if (stream)
1041     {
1042         recordAttr->recordSize = end_offset - start_offset;
1043         zebraExplain_recordBytesIncrement(zh->reg->zei,
1044                                           recordAttr->recordSize);
1045     }
1046
1047     /* set run-number for this record */
1048     recordAttr->runNumber =
1049         zebraExplain_runNumberIncrement (zh->reg->zei, 0);
1050
1051     /* update store data */
1052     xfree (rec->info[recInfo_storeData]);
1053
1054     /* update store data */
1055     if (zh->store_data_buf)
1056     {
1057         rec->size[recInfo_storeData] = zh->store_data_size;
1058         rec->info[recInfo_storeData] = zh->store_data_buf;
1059         zh->store_data_buf = 0;
1060         recordAttr->recordSize = zh->store_data_size;
1061     }
1062     else if (zh->m_store_data)
1063     {
1064         off_t cur_offset = stream->tellf(stream);
1065
1066         rec->size[recInfo_storeData] = recordAttr->recordSize;
1067         rec->info[recInfo_storeData] = (char *)
1068             xmalloc (recordAttr->recordSize);
1069         stream->seekf(stream, start_offset);
1070         stream->readf(stream, rec->info[recInfo_storeData],
1071                       recordAttr->recordSize);
1072         stream->seekf(stream, cur_offset);
1073     }
1074     else
1075     {
1076         rec->info[recInfo_storeData] = NULL;
1077         rec->size[recInfo_storeData] = 0;
1078     }
1079     /* update database name */
1080     xfree (rec->info[recInfo_databaseName]);
1081     rec->info[recInfo_databaseName] =
1082         rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
1083
1084     /* update offset */
1085     recordAttr->recordOffset = start_offset;
1086
1087     /* commit this record */
1088     rec_put (zh->reg->records, &rec);
1089     logRecord(zh);
1090     return ZEBRA_OK;
1091 }
1092
1093 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
1094 {
1095     ZebraHandle zh = (ZebraHandle) handle;
1096     struct recExtractCtrl extractCtrl;
1097
1098     if (zebraExplain_curDatabase (zh->reg->zei,
1099                                   rec->info[recInfo_databaseName]))
1100     {
1101         abort();
1102         if (zebraExplain_newDatabase (zh->reg->zei,
1103                                       rec->info[recInfo_databaseName], 0))
1104             abort ();
1105     }
1106
1107     zebra_rec_keys_reset(zh->reg->keys);
1108     zebra_rec_keys_reset(zh->reg->sortKeys);
1109
1110     extractCtrl.init = extract_init;
1111     extractCtrl.tokenAdd = extract_token_add;
1112     extractCtrl.schemaAdd = extract_schema_add;
1113     extractCtrl.dh = zh->reg->dh;
1114
1115     init_extractCtrl(zh, &extractCtrl);
1116
1117     extractCtrl.flagShowRecords = 0;
1118     extractCtrl.match_criteria[0] = '\0';
1119     extractCtrl.staticrank = 0;
1120     extractCtrl.action = action_update;
1121
1122     extractCtrl.handle = handle;
1123     extractCtrl.first_record = 1;
1124
1125     extract_set_store_data_prepare(&extractCtrl);
1126
1127     if (n)
1128         grs_extract_tree(&extractCtrl, n);
1129
1130     if (rec->size[recInfo_delKeys])
1131     {
1132         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
1133
1134         zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
1135
1136         zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
1137                                rec->size[recInfo_delKeys],
1138                                0);
1139 #if FLUSH2
1140         extract_flush_record_keys2(zh, rec->sysno,
1141                                    zh->reg->keys, 0, delkeys, 0);
1142 #else
1143         extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
1144         extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
1145 #endif
1146         zebra_rec_keys_close(delkeys);
1147
1148         zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
1149                                rec->size[recInfo_sortKeys],
1150                                0);
1151
1152         extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
1153         zebra_rec_keys_close(sortkeys);
1154     }
1155     else
1156     {
1157 #if FLUSH2
1158         extract_flush_record_keys2(zh, rec->sysno, zh->reg->keys, 0, 0, 0);
1159 #else
1160         extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
1161 #endif
1162     }
1163     extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
1164
1165     xfree (rec->info[recInfo_delKeys]);
1166     zebra_rec_keys_get_buf(zh->reg->keys,
1167                            &rec->info[recInfo_delKeys],
1168                            &rec->size[recInfo_delKeys]);
1169
1170     xfree (rec->info[recInfo_sortKeys]);
1171     zebra_rec_keys_get_buf(zh->reg->sortKeys,
1172                            &rec->info[recInfo_sortKeys],
1173                            &rec->size[recInfo_sortKeys]);
1174     return ZEBRA_OK;
1175 }
1176
1177 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
1178                           zebra_rec_keys_t reckeys,
1179                           int level)
1180 {
1181     if (zebra_rec_keys_rewind(reckeys))
1182     {
1183         size_t slen;
1184         const char *str;
1185         struct it_key key;
1186         NMEM nmem = nmem_create();
1187
1188         while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
1189         {
1190             char keystr[200]; /* room for zints to print */
1191             char *dst_term = 0;
1192             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1193             int index_type, i;
1194             const char *string_index;
1195
1196             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1197                                     0/* db */, &string_index);
1198             assert(index_type);
1199             zebra_term_untrans_iconv(zh, nmem, index_type,
1200                                      &dst_term, str);
1201             *keystr = '\0';
1202             for (i = 0; i<key.len; i++)
1203             {
1204                 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key.mem[i]);
1205             }
1206
1207             if (*str < CHR_BASE_CHAR)
1208             {
1209                 int i;
1210                 char dst_buf[200]; /* room for special chars */
1211
1212                 strcpy(dst_buf , "?");
1213
1214                 if (!strcmp(str, ""))
1215                     strcpy(dst_buf, "alwaysmatches");
1216                 if (!strcmp(str, FIRST_IN_FIELD_STR))
1217                     strcpy(dst_buf, "firstinfield");
1218                 else if (!strcmp(str, CHR_UNKNOWN))
1219                     strcpy(dst_buf, "unknown");
1220                 else if (!strcmp(str, CHR_SPACE))
1221                     strcpy(dst_buf, "space");
1222
1223                 for (i = 0; i<slen; i++)
1224                 {
1225                     sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
1226                 }
1227                 yaz_log(level, "%s%c %s %s", keystr, index_type,
1228                         string_index, dst_buf);
1229
1230             }
1231             else
1232                 yaz_log(level, "%s%c %s \"%s\"", keystr, index_type,
1233                         string_index, dst_term);
1234
1235             nmem_reset(nmem);
1236         }
1237         nmem_destroy(nmem);
1238     }
1239 }
1240
1241 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
1242                              zebra_rec_keys_t reckeys)
1243 {
1244     ZebraExplainInfo zei = zh->reg->zei;
1245     struct ord_stat {
1246         int no;
1247         int ord;
1248         struct ord_stat *next;
1249     };
1250
1251     if (zebra_rec_keys_rewind(reckeys))
1252     {
1253         struct ord_stat *ord_list = 0;
1254         struct ord_stat *p;
1255         size_t slen;
1256         const char *str;
1257         struct it_key key_in;
1258         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1259         {
1260             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1261
1262             for (p = ord_list; p ; p = p->next)
1263                 if (p->ord == ord)
1264                 {
1265                     p->no++;
1266                     break;
1267                 }
1268             if (!p)
1269             {
1270                 p = xmalloc(sizeof(*p));
1271                 p->no = 1;
1272                 p->ord = ord;
1273                 p->next = ord_list;
1274                 ord_list = p;
1275             }
1276         }
1277
1278         p = ord_list;
1279         while (p)
1280         {
1281             struct ord_stat *p1 = p;
1282
1283             if (is_insert)
1284                 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
1285             else
1286                 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1287             p = p->next;
1288             xfree(p1);
1289         }
1290     }
1291 }
1292
1293 void extract_flush_record_keys2(ZebraHandle zh, zint sysno,
1294                                 zebra_rec_keys_t ins_keys, zint ins_rank,
1295                                 zebra_rec_keys_t del_keys, zint del_rank)
1296 {
1297     ZebraExplainInfo zei = zh->reg->zei;
1298     int normal = 0;
1299     int optimized = 0;
1300
1301     if (!zh->reg->key_block)
1302     {
1303         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1304         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1305         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1306         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1307     }
1308
1309     if (ins_keys)
1310     {
1311         extract_rec_keys_adjust(zh, 1, ins_keys);
1312         if (!del_keys)
1313             zebraExplain_recordCountIncrement (zei, 1);
1314         zebra_rec_keys_rewind(ins_keys);
1315     }
1316     if (del_keys)
1317     {
1318         extract_rec_keys_adjust(zh, 0, del_keys);
1319         if (!ins_keys)
1320             zebraExplain_recordCountIncrement (zei, -1);
1321         zebra_rec_keys_rewind(del_keys);
1322     }
1323
1324     while (1)
1325     {
1326         size_t del_slen;
1327         const char *del_str;
1328         struct it_key del_key_in;
1329         int del = 0;
1330
1331         size_t ins_slen;
1332         const char *ins_str;
1333         struct it_key ins_key_in;
1334         int ins = 0;
1335
1336         if (del_keys)
1337             del = zebra_rec_keys_read(del_keys, &del_str, &del_slen,
1338                                       &del_key_in);
1339         if (ins_keys)
1340             ins = zebra_rec_keys_read(ins_keys, &ins_str, &ins_slen,
1341                                       &ins_key_in);
1342
1343         if (del && ins && ins_rank == del_rank
1344             && !key_compare(&del_key_in, &ins_key_in)
1345             && ins_slen == del_slen && !memcmp(del_str, ins_str, del_slen))
1346         {
1347             optimized++;
1348             continue;
1349         }
1350         if (!del && !ins)
1351             break;
1352
1353         normal++;
1354         if (del)
1355             key_block_write(zh->reg->key_block, sysno,
1356                             &del_key_in, 0, del_str, del_slen,
1357                             del_rank, zh->m_staticrank);
1358         if (ins)
1359             key_block_write(zh->reg->key_block, sysno,
1360                             &ins_key_in, 1, ins_str, ins_slen,
1361                             ins_rank, zh->m_staticrank);
1362     }
1363     yaz_log(log_level_extract, "normal=%d optimized=%d", normal, optimized);
1364 }
1365
1366 void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd,
1367                                zebra_rec_keys_t reckeys,
1368                                zint staticrank)
1369 {
1370     ZebraExplainInfo zei = zh->reg->zei;
1371
1372     extract_rec_keys_adjust(zh, cmd, reckeys);
1373
1374     if (log_level_details)
1375     {
1376         yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1377                 sysno, cmd ? "insert" : "delete");
1378         extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1379     }
1380
1381     if (!zh->reg->key_block)
1382     {
1383         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1384         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1385         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1386         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1387     }
1388     zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1389
1390 #if 0
1391     yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd);
1392     print_rec_keys(zh, reckeys);
1393 #endif
1394     if (zebra_rec_keys_rewind(reckeys))
1395     {
1396         size_t slen;
1397         const char *str;
1398         struct it_key key_in;
1399         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1400         {
1401             key_block_write(zh->reg->key_block, sysno,
1402                             &key_in, cmd, str, slen,
1403                             staticrank, zh->m_staticrank);
1404         }
1405     }
1406 }
1407
1408 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
1409                                      zebra_rec_keys_t reckeys,
1410                                      zebra_snippets *snippets)
1411 {
1412     NMEM nmem = nmem_create();
1413     if (zebra_rec_keys_rewind(reckeys))
1414     {
1415         const char *str;
1416         size_t slen;
1417         struct it_key key;
1418         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1419         {
1420             char *dst_term = 0;
1421             int ord;
1422             zint seqno;
1423             int index_type;
1424
1425             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1426             seqno = key.mem[key.len-1];
1427             ord = CAST_ZINT_TO_INT(key.mem[0]);
1428
1429             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1430                                     0/* db */, 0 /* string_index */);
1431             assert(index_type);
1432             zebra_term_untrans_iconv(zh, nmem, index_type,
1433                                      &dst_term, str);
1434             zebra_snippets_append(snippets, seqno, 0, ord, dst_term);
1435             nmem_reset(nmem);
1436         }
1437     }
1438     nmem_destroy(nmem);
1439     return ZEBRA_OK;
1440 }
1441
1442 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1443 {
1444     yaz_log(YLOG_LOG, "print_rec_keys");
1445     if (zebra_rec_keys_rewind(reckeys))
1446     {
1447         const char *str;
1448         size_t slen;
1449         struct it_key key;
1450         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1451         {
1452             char dst_buf[IT_MAX_WORD];
1453             zint seqno;
1454             int index_type;
1455             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1456             const char *db = 0;
1457             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1458
1459             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1460
1461             seqno = key.mem[key.len-1];
1462
1463             zebra_term_untrans(zh, index_type, dst_buf, str);
1464
1465             yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1466                     " term=%s", ord, seqno, dst_buf);
1467         }
1468     }
1469 }
1470
1471 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1472                                      const char *str, int length)
1473 {
1474     struct it_key key;
1475     ZebraHandle zh = p->extractCtrl->handle;
1476     ZebraExplainInfo zei = zh->reg->zei;
1477     int ch, i;
1478
1479     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1480     if (ch < 0)
1481         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1482
1483     i = 0;
1484     key.mem[i++] = ch;
1485     key.mem[i++] = p->record_id;
1486     key.mem[i++] = p->section_id;
1487
1488     if (zh->m_segment_indexing)
1489         key.mem[i++] = p->segment;
1490     key.mem[i++] = p->seqno;
1491     key.len = i;
1492
1493     zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1494 }
1495
1496 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1497 {
1498     struct it_key key;
1499     ZebraHandle zh = p->extractCtrl->handle;
1500     ZebraExplainInfo zei = zh->reg->zei;
1501     int ch;
1502     zinfo_index_category_t cat = zinfo_index_category_sort;
1503
1504     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1505     if (ch < 0)
1506         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1507     key.len = 2;
1508     key.mem[0] = ch;
1509     key.mem[1] = p->record_id;
1510
1511     zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1512 }
1513
1514 static void extract_add_staticrank_string(RecWord *p,
1515                                           const char *str, int length)
1516 {
1517     char valz[40];
1518     struct recExtractCtrl *ctrl = p->extractCtrl;
1519
1520     if (length > sizeof(valz)-1)
1521         length = sizeof(valz)-1;
1522
1523     memcpy(valz, str, length);
1524     valz[length] = '\0';
1525     ctrl->staticrank = atozint(valz);
1526 }
1527
1528 static void extract_add_string(RecWord *p, const char *string, int length)
1529 {
1530     ZebraHandle zh = p->extractCtrl->handle;
1531     assert (length > 0);
1532
1533     if (!p->index_name)
1534         return;
1535
1536     if (zebra_maps_is_index(zh->reg->zebra_maps, p->index_type))
1537     {
1538         extract_add_index_string(p, zinfo_index_category_index,
1539                                  string, length);
1540         if (zebra_maps_is_alwaysmatches(zh->reg->zebra_maps, p->index_type))
1541         {
1542             RecWord word;
1543             memcpy(&word, p, sizeof(word));
1544
1545             word.seqno = 1;
1546             extract_add_index_string(
1547                 &word, zinfo_index_category_alwaysmatches, "", 0);
1548         }
1549     }
1550     else if (zebra_maps_is_sort(zh->reg->zebra_maps, p->index_type))
1551     {
1552         extract_add_sort_string(p, string, length);
1553     }
1554     else if (zebra_maps_is_staticrank(zh->reg->zebra_maps, p->index_type))
1555     {
1556         extract_add_staticrank_string(p, string, length);
1557     }
1558 }
1559
1560 static void extract_add_incomplete_field(RecWord *p)
1561 {
1562     ZebraHandle zh = p->extractCtrl->handle;
1563     const char *b = p->term_buf;
1564     int remain = p->term_len;
1565     int first = 1;
1566     const char **map = 0;
1567
1568     if (remain > 0)
1569         map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1570
1571     while (map)
1572     {
1573         char buf[IT_MAX_WORD+1];
1574         int i, remain;
1575
1576         /* Skip spaces */
1577         while (map && *map && **map == *CHR_SPACE)
1578         {
1579             remain = p->term_len - (b - p->term_buf);
1580             if (remain > 0)
1581                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b,
1582                                        remain, 0);
1583             else
1584                 map = 0;
1585         }
1586         if (!map)
1587             break;
1588         i = 0;
1589         while (map && *map && **map != *CHR_SPACE)
1590         {
1591             const char *cp = *map;
1592
1593             while (i < IT_MAX_WORD && *cp)
1594                 buf[i++] = *(cp++);
1595             remain = p->term_len - (b - p->term_buf);
1596             if (remain > 0)
1597                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1598             else
1599                 map = 0;
1600         }
1601         if (!i)
1602             return;
1603
1604         if (first)
1605         {
1606             first = 0;
1607             if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type))
1608             {
1609                 /* first in field marker */
1610                 extract_add_string(p, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1611                 p->seqno++;
1612             }
1613         }
1614         extract_add_string (p, buf, i);
1615         p->seqno++;
1616     }
1617 }
1618
1619 static void extract_add_complete_field (RecWord *p)
1620 {
1621     ZebraHandle zh = p->extractCtrl->handle;
1622     const char *b = p->term_buf;
1623     char buf[IT_MAX_WORD+1];
1624     const char **map = 0;
1625     int i = 0, remain = p->term_len;
1626
1627     if (remain > 0)
1628         map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, remain, 1);
1629
1630     while (remain > 0 && i < IT_MAX_WORD)
1631     {
1632         while (map && *map && **map == *CHR_SPACE)
1633         {
1634             remain = p->term_len - (b - p->term_buf);
1635
1636             if (remain > 0)
1637             {
1638                 int first = i ? 0 : 1;  /* first position */
1639                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, first);
1640             }
1641             else
1642                 map = 0;
1643         }
1644         if (!map)
1645             break;
1646
1647         if (i && i < IT_MAX_WORD)
1648             buf[i++] = *CHR_SPACE;
1649         while (map && *map && **map != *CHR_SPACE)
1650         {
1651             const char *cp = *map;
1652
1653             if (**map == *CHR_CUT)
1654             {
1655                 i = 0;
1656             }
1657             else
1658             {
1659                 if (i >= IT_MAX_WORD)
1660                     break;
1661                 while (i < IT_MAX_WORD && *cp)
1662                     buf[i++] = *(cp++);
1663             }
1664             remain = p->term_len  - (b - p->term_buf);
1665             if (remain > 0)
1666             {
1667                 map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b,
1668                                         remain, 0);
1669             }
1670             else
1671                 map = 0;
1672         }
1673     }
1674     if (!i)
1675         return;
1676     extract_add_string (p, buf, i);
1677 }
1678
1679 static void extract_token_add(RecWord *p)
1680 {
1681     ZebraHandle zh = p->extractCtrl->handle;
1682     WRBUF wrbuf;
1683
1684     if (log_level_details)
1685     {
1686         yaz_log(log_level_details, "extract_token_add "
1687                 "type=%c index=%s seqno=" ZINT_FORMAT " s=%.*s",
1688                 p->index_type, p->index_name,
1689                 p->seqno, p->term_len, p->term_buf);
1690     }
1691     if ((wrbuf = zebra_replace(zh->reg->zebra_maps, p->index_type, 0,
1692                                p->term_buf, p->term_len)))
1693     {
1694         p->term_buf = wrbuf_buf(wrbuf);
1695         p->term_len = wrbuf_len(wrbuf);
1696     }
1697     if (zebra_maps_is_complete (zh->reg->zebra_maps, p->index_type))
1698         extract_add_complete_field (p);
1699     else
1700         extract_add_incomplete_field(p);
1701 }
1702
1703 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1704                                       void *buf, size_t sz)
1705 {
1706     ZebraHandle zh = (ZebraHandle) p->handle;
1707
1708     xfree(zh->store_data_buf);
1709     zh->store_data_buf = 0;
1710     zh->store_data_size = 0;
1711     if (buf && sz)
1712     {
1713         zh->store_data_buf = xmalloc(sz);
1714         zh->store_data_size = sz;
1715         memcpy(zh->store_data_buf, buf, sz);
1716     }
1717 }
1718
1719 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1720 {
1721     ZebraHandle zh = (ZebraHandle) p->handle;
1722     xfree(zh->store_data_buf);
1723     zh->store_data_buf = 0;
1724     zh->store_data_size = 0;
1725     p->setStoreData = extract_set_store_data_cb;
1726 }
1727
1728 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1729 {
1730     ZebraHandle zh = (ZebraHandle) p->handle;
1731     zebraExplain_addSchema (zh->reg->zei, oid);
1732 }
1733
1734 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1735                              int cmd, zebra_rec_keys_t reckeys)
1736 {
1737 #if 0
1738     yaz_log(YLOG_LOG, "extract_flush_sort_keys cmd=%d sysno=" ZINT_FORMAT,
1739             cmd, sysno);
1740     extract_rec_keys_log(zh, cmd, reckeys, YLOG_LOG);
1741 #endif
1742
1743     if (zebra_rec_keys_rewind(reckeys))
1744     {
1745         zebra_sort_index_t si = zh->reg->sort_index;
1746         size_t slen;
1747         const char *str;
1748         struct it_key key_in;
1749
1750         zebra_sort_sysno(si, sysno);
1751
1752         while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1753         {
1754             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1755
1756             zebra_sort_type(si, ord);
1757             if (cmd == 1)
1758                 zebra_sort_add(si, str, slen);
1759             else
1760                 zebra_sort_delete(si);
1761         }
1762     }
1763 }
1764
1765 /*
1766  * Local variables:
1767  * c-basic-offset: 4
1768  * indent-tabs-mode: nil
1769  * End:
1770  * vim: shiftwidth=4 tabstop=8 expandtab
1771  */
1772