index/extract.c

   1 /* $Id: extract.c,v 1.253 2007-03-14 14:16:14 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #endif
  29 #if HAVE_UNISTD_H
  30 #include <unistd.h>
  31 #endif
  32 #include <fcntl.h>
  33
  34 #include "index.h"
  35 #include "orddict.h"
  36 #include <direntz.h>
  37 #include <charmap.h>
  38
  39 static int log_level_extract = 0;
  40 static int log_level_details = 0;
  41 static int log_level_initialized = 0;
  42
  43 static void zebra_init_log_level(void)
  44 {
  45     if (!log_level_initialized)
  46     {
  47         log_level_initialized = 1;
  48
  49         log_level_extract = yaz_log_module_level("extract");
  50         log_level_details = yaz_log_module_level("indexdetails");
  51     }
  52 }
  53
  54 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
  55                                       int cmd, zebra_rec_keys_t reckeys,
  56                                       zint staticrank);
  57 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
  58                                     int cmd, zebra_rec_keys_t skp);
  59 static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid);
  60 static void extract_token_add (RecWord *p);
  61
  62 static void check_log_limit(ZebraHandle zh)
  63 {
  64     if (zh->records_processed + zh->records_skipped == zh->m_file_verbose_limit)
  65     {
  66         yaz_log(YLOG_LOG, "More than %d file log entries. Omitting rest",
  67                 zh->m_file_verbose_limit);
  68     }
  69 }
  70
  71 static void logRecord (ZebraHandle zh)
  72 {
  73     check_log_limit(zh);
  74     ++zh->records_processed;
  75     if (!(zh->records_processed % 1000))
  76     {
  77         yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
  78                 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
  79                 zh->records_processed, zh->records_inserted,
  80                 zh->records_updated, zh->records_deleted);
  81     }
  82 }
  83
  84 static void extract_add_index_string (RecWord *p,
  85                                       zinfo_index_category_t cat,
  86                                       const char *str, int length);
  87
  88 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
  89
  90 static void extract_init(struct recExtractCtrl *p, RecWord *w)
  91 {
  92     w->seqno = 1;
  93     w->index_name = "any";
  94     w->index_type = 'w';
  95     w->extractCtrl = p;
  96     w->record_id = 0;
  97     w->section_id = 0;
  98     w->segment = 0;
  99 }
 100
 101 static void searchRecordKey(ZebraHandle zh,
 102                             zebra_rec_keys_t reckeys,
 103                             const char *index_name,
 104                             const char **ws, int ws_length)
 105 {
 106     int i;
 107     int ch = -1;
 108     zinfo_index_category_t cat = zinfo_index_category_index;
 109
 110     for (i = 0; i<ws_length; i++)
 111         ws[i] = NULL;
 112
 113     if (ch < 0)
 114         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, '0', index_name);
 115     if (ch < 0)
 116         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'p', index_name);
 117     if (ch < 0)
 118         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'w', index_name);
 119
 120     if (ch < 0)
 121         return ;
 122
 123     if (zebra_rec_keys_rewind(reckeys))
 124     {
 125         zint startSeq = -1;
 126         const char *str;
 127         size_t slen;
 128         struct it_key key;
 129         zint seqno;
 130         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
 131         {
 132             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
 133
 134             seqno = key.mem[key.len-1];
 135
 136             if (key.mem[0] == ch)
 137             {
 138                 zint woff;
 139
 140                 if (startSeq == -1)
 141                     startSeq = seqno;
 142                 woff = seqno - startSeq;
 143                 if (woff >= 0 && woff < ws_length)
 144                     ws[woff] = str;
 145             }
 146         }
 147     }
 148 }
 149
 150 #define FILE_MATCH_BLANK "\t "
 151
 152 static char *get_match_from_spec(ZebraHandle zh,
 153                           zebra_rec_keys_t reckeys,
 154                           const char *fname, const char *spec)
 155 {
 156     static char dstBuf[2048];      /* static here ??? */
 157     char *dst = dstBuf;
 158     const char *s = spec;
 159
 160     while (1)
 161     {
 162         for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
 163             ;
 164         if (!*s)
 165             break;
 166         if (*s == '(')
 167         {
 168             const char *ws[32];
 169             char attset_str[64], attname_str[64];
 170             int i;
 171             int first = 1;
 172
 173             for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 174                 ;
 175             for (i = 0; *s && *s != ',' && *s != ')' &&
 176                      !strchr(FILE_MATCH_BLANK, *s); s++)
 177                 if (i+1 < sizeof(attset_str))
 178                     attset_str[i++] = *s;
 179             attset_str[i] = '\0';
 180
 181             for (; strchr(FILE_MATCH_BLANK, *s); s++)
 182                 ;
 183             if (*s != ',')
 184                 strcpy(attname_str, attset_str);
 185             else
 186             {
 187                 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 188                     ;
 189                 for (i = 0; *s && *s != ')' &&
 190                          !strchr(FILE_MATCH_BLANK, *s); s++)
 191                     if (i+1 < sizeof(attname_str))
 192                         attname_str[i++] = *s;
 193                 attname_str[i] = '\0';
 194             }
 195
 196             searchRecordKey (zh, reckeys, attname_str, ws, 32);
 197
 198             if (*s != ')')
 199             {
 200                 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
 201                       spec, zh->m_group ? zh->m_group : "none");
 202                 return NULL;
 203             }
 204             s++;
 205
 206             for (i = 0; i<32; i++)
 207                 if (ws[i])
 208                 {
 209                     if (first)
 210                     {
 211                         *dst++ = ' ';
 212                         first = 0;
 213                     }
 214                     strcpy (dst, ws[i]);
 215                     dst += strlen(ws[i]);
 216                 }
 217             if (first)
 218             {
 219                 yaz_log (YLOG_WARN, "Record didn't contain match"
 220                       " fields in (%s,%s)", attset_str, attname_str);
 221                 return NULL;
 222             }
 223         }
 224         else if (*s == '$')
 225         {
 226             int spec_len;
 227             char special[64];
 228             const char *spec_src = NULL;
 229             const char *s1 = ++s;
 230             while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
 231                 s1++;
 232
 233             spec_len = s1 - s;
 234             if (spec_len > sizeof(special)-1)
 235                 spec_len = sizeof(special)-1;
 236             memcpy (special, s, spec_len);
 237             special[spec_len] = '\0';
 238             s = s1;
 239
 240             if (!strcmp (special, "group"))
 241                 spec_src = zh->m_group;
 242             else if (!strcmp (special, "database"))
 243                 spec_src = zh->basenames[0];
 244             else if (!strcmp (special, "filename")) {
 245                 spec_src = fname;
 246             }
 247             else if (!strcmp (special, "type"))
 248                 spec_src = zh->m_record_type;
 249             else
 250                 spec_src = NULL;
 251             if (spec_src)
 252             {
 253                 strcpy (dst, spec_src);
 254                 dst += strlen (spec_src);
 255             }
 256         }
 257         else if (*s == '\"' || *s == '\'')
 258         {
 259             int stopMarker = *s++;
 260             char tmpString[64];
 261             int i = 0;
 262
 263             while (*s && *s != stopMarker)
 264             {
 265                 if (i+1 < sizeof(tmpString))
 266                     tmpString[i++] = *s++;
 267             }
 268             if (*s)
 269                 s++;
 270             tmpString[i] = '\0';
 271             strcpy (dst, tmpString);
 272             dst += strlen (tmpString);
 273         }
 274         else
 275         {
 276             yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
 277                   spec, zh->m_group ? zh->m_group : "none");
 278             return NULL;
 279         }
 280         *dst++ = 1;
 281     }
 282     if (dst == dstBuf)
 283     {
 284         yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
 285               fname, zh->m_group ? zh->m_group : "none");
 286         return NULL;
 287     }
 288     *dst = '\0';
 289     return dstBuf;
 290 }
 291
 292 struct recordLogInfo {
 293     const char *fname;
 294     int recordOffset;
 295     struct recordGroup *rGroup;
 296 };
 297
 298 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
 299 {
 300     int i;
 301     for (i = 0; i<256; i++)
 302     {
 303         if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
 304             ctrl->seqno[i] = 1;
 305         else
 306             ctrl->seqno[i] = 0;
 307     }
 308     ctrl->flagShowRecords = !zh->m_flag_rw;
 309 }
 310
 311 static void all_matches_add(struct recExtractCtrl *ctrl)
 312 {
 313     RecWord word;
 314     extract_init(ctrl, &word);
 315     word.index_name = "_ALLRECORDS";
 316     word.index_type = 'w';
 317     word.seqno = 1;
 318     extract_add_index_string (&word, zinfo_index_category_alwaysmatches,
 319                               "", 0);
 320 }
 321
 322 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
 323                                        struct ZebraRecStream *stream,
 324                                        enum zebra_recctrl_action_t action,
 325                                        int test_mode,
 326                                        const char *recordType,
 327                                        zint *sysno,
 328                                        const char *match_criteria,
 329                                        const char *fname,
 330                                        RecType recType,
 331                                        void *recTypeClientData);
 332
 333
 334 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
 335                              int deleteFlag)
 336 {
 337     ZEBRA_RES r = ZEBRA_OK;
 338     int i, fd;
 339     char gprefix[128];
 340     char ext[128];
 341     char ext_res[128];
 342     struct file_read_info *fi = 0;
 343     const char *original_record_type = 0;
 344     RecType recType;
 345     void *recTypeClientData;
 346     struct ZebraRecStream stream, *streamp;
 347
 348     zebra_init_log_level();
 349
 350     if (!zh->m_group || !*zh->m_group)
 351         *gprefix = '\0';
 352     else
 353         sprintf (gprefix, "%s.", zh->m_group);
 354
 355     yaz_log(log_level_extract, "zebra_extract_file %s", fname);
 356
 357     /* determine file extension */
 358     *ext = '\0';
 359     for (i = strlen(fname); --i >= 0; )
 360         if (fname[i] == '/')
 361             break;
 362         else if (fname[i] == '.')
 363         {
 364             strcpy (ext, fname+i+1);
 365             break;
 366         }
 367     /* determine file type - depending on extension */
 368     original_record_type = zh->m_record_type;
 369     if (!zh->m_record_type)
 370     {
 371         sprintf (ext_res, "%srecordType.%s", gprefix, ext);
 372         zh->m_record_type = res_get (zh->res, ext_res);
 373     }
 374     if (!zh->m_record_type)
 375     {
 376         check_log_limit(zh);
 377         if (zh->records_processed + zh->records_skipped
 378             < zh->m_file_verbose_limit)
 379             yaz_log (YLOG_LOG, "? %s", fname);
 380         zh->records_skipped++;
 381         return 0;
 382     }
 383     /* determine match criteria */
 384     if (!zh->m_record_id)
 385     {
 386         sprintf (ext_res, "%srecordId.%s", gprefix, ext);
 387         zh->m_record_id = res_get (zh->res, ext_res);
 388     }
 389
 390     if (!(recType =
 391           recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
 392                           &recTypeClientData)))
 393     {
 394         yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
 395         return ZEBRA_FAIL;
 396     }
 397
 398     switch(recType->version)
 399     {
 400     case 0:
 401         break;
 402     default:
 403         yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
 404     }
 405     if (sysno && deleteFlag)
 406     {
 407         streamp = 0;
 408         fi = 0;
 409     }
 410     else
 411     {
 412         char full_rep[1024];
 413
 414         if (zh->path_reg && !yaz_is_abspath (fname))
 415         {
 416             strcpy (full_rep, zh->path_reg);
 417             strcat (full_rep, "/");
 418             strcat (full_rep, fname);
 419         }
 420         else
 421             strcpy (full_rep, fname);
 422
 423         if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
 424         {
 425             yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
 426             zh->m_record_type = original_record_type;
 427             return ZEBRA_FAIL;
 428         }
 429         streamp = &stream;
 430         zebra_create_stream_fd(streamp, fd, 0);
 431     }
 432     r = zebra_extract_records_stream(zh, streamp,
 433                                      deleteFlag ?
 434                                      action_delete : action_update,
 435                                      0, /* tst_mode */
 436                                      zh->m_record_type,
 437                                      sysno,
 438                                      0, /*match_criteria */
 439                                      fname,
 440                                      recType, recTypeClientData);
 441     if (streamp)
 442         stream.destroy(streamp);
 443     zh->m_record_type = original_record_type;
 444     return r;
 445 }
 446
 447 /*
 448   If sysno is provided, then it's used to identify the reocord.
 449   If not, and match_criteria is provided, then sysno is guessed
 450   If not, and a record is provided, then sysno is got from there
 451
 452  */
 453
 454 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
 455                                       const char *buf, size_t buf_size,
 456                                       enum zebra_recctrl_action_t action,
 457                                       int test_mode,
 458                                       const char *recordType,
 459                                       zint *sysno,
 460                                       const char *match_criteria,
 461                                       const char *fname)
 462 {
 463     struct ZebraRecStream stream;
 464     ZEBRA_RES res;
 465     void *clientData;
 466     RecType recType = 0;
 467
 468     if (recordType && *recordType)
 469     {
 470         yaz_log(log_level_extract,
 471                 "Record type explicitly specified: %s", recordType);
 472         recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
 473                                   &clientData);
 474     }
 475     else
 476     {
 477         if (!(zh->m_record_type))
 478         {
 479             yaz_log (YLOG_WARN, "No such record type defined");
 480             return ZEBRA_FAIL;
 481         }
 482         yaz_log(log_level_extract, "Get record type from rgroup: %s",
 483                 zh->m_record_type);
 484         recType = recType_byName (zh->reg->recTypes, zh->res,
 485                                   zh->m_record_type, &clientData);
 486         recordType = zh->m_record_type;
 487     }
 488
 489     if (!recType)
 490     {
 491         yaz_log (YLOG_WARN, "No such record type: %s", recordType);
 492         return ZEBRA_FAIL;
 493     }
 494
 495     zebra_create_stream_mem(&stream, buf, buf_size);
 496
 497     res = zebra_extract_records_stream(zh, &stream,
 498                                        action,
 499                                        test_mode,
 500                                        recordType,
 501                                        sysno,
 502                                        match_criteria,
 503                                        fname,
 504                                        recType, clientData);
 505     stream.destroy(&stream);
 506     return res;
 507 }
 508
 509 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
 510                                        struct ZebraRecStream *stream,
 511                                        enum zebra_recctrl_action_t action,
 512                                        int test_mode,
 513                                        const char *recordType,
 514                                        zint *sysno,
 515                                        const char *match_criteria,
 516                                        const char *fname,
 517                                        RecType recType,
 518                                        void *recTypeClientData)
 519 {
 520     ZEBRA_RES res = ZEBRA_OK;
 521     while (1)
 522     {
 523         int more = 0;
 524         res = zebra_extract_record_stream(zh, stream,
 525                                           action,
 526                                           test_mode,
 527                                           recordType,
 528                                           sysno,
 529                                           match_criteria,
 530                                           fname,
 531                                           recType, recTypeClientData, &more);
 532         if (!more)
 533         {
 534             res = ZEBRA_OK;
 535             break;
 536         }
 537         if (res != ZEBRA_OK)
 538             break;
 539         if (sysno)
 540             break;
 541     }
 542     return res;
 543 }
 544
 545
 546 ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
 547                                       struct ZebraRecStream *stream,
 548                                       enum zebra_recctrl_action_t action,
 549                                       int test_mode,
 550                                       const char *recordType,
 551                                       zint *sysno,
 552                                       const char *match_criteria,
 553                                       const char *fname,
 554                                       RecType recType,
 555                                       void *recTypeClientData,
 556                                       int *more)
 557
 558 {
 559     zint sysno0 = 0;
 560     RecordAttr *recordAttr;
 561     struct recExtractCtrl extractCtrl;
 562     int r;
 563     const char *matchStr = 0;
 564     Record rec;
 565     off_t start_offset = 0, end_offset = 0;
 566     const char *pr_fname = fname;  /* filename to print .. */
 567     int show_progress = zh->records_processed + zh->records_skipped
 568         < zh->m_file_verbose_limit ? 1:0;
 569
 570     zebra_init_log_level();
 571
 572     if (!pr_fname)
 573         pr_fname = "<no file>";  /* make it printable if file is omitted */
 574
 575     zebra_rec_keys_reset(zh->reg->keys);
 576     zebra_rec_keys_reset(zh->reg->sortKeys);
 577
 578     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
 579     {
 580         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
 581                                       zh->m_explain_database))
 582             return ZEBRA_FAIL;
 583     }
 584
 585     if (stream)
 586     {
 587         off_t null_offset = 0;
 588         extractCtrl.stream = stream;
 589
 590         start_offset = stream->tellf(stream);
 591
 592         extractCtrl.first_record = start_offset ? 0 : 1;
 593
 594         stream->endf(stream, &null_offset);;
 595
 596         extractCtrl.init = extract_init;
 597         extractCtrl.tokenAdd = extract_token_add;
 598         extractCtrl.schemaAdd = extract_schema_add;
 599         extractCtrl.dh = zh->reg->dh;
 600         extractCtrl.handle = zh;
 601         extractCtrl.match_criteria[0] = '\0';
 602         extractCtrl.staticrank = 0;
 603         extractCtrl.action = action;
 604
 605         init_extractCtrl(zh, &extractCtrl);
 606
 607         extract_set_store_data_prepare(&extractCtrl);
 608
 609         r = (*recType->extract)(recTypeClientData, &extractCtrl);
 610
 611         yaz_log(YLOG_LOG, "Old action=%d new action=%d", action,
 612                 extractCtrl.action);
 613         if (action == action_update)
 614         {
 615             action = extractCtrl.action;
 616         }
 617
 618         switch (r)
 619         {
 620         case RECCTRL_EXTRACT_EOF:
 621             return ZEBRA_FAIL;
 622         case RECCTRL_EXTRACT_ERROR_GENERIC:
 623             /* error occured during extraction ... */
 624             yaz_log (YLOG_WARN, "extract error: generic");
 625             return ZEBRA_FAIL;
 626         case RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER:
 627             /* error occured during extraction ... */
 628             yaz_log (YLOG_WARN, "extract error: no such filter");
 629             return ZEBRA_FAIL;
 630         case RECCTRL_EXTRACT_SKIP:
 631             if (show_progress)
 632                 yaz_log (YLOG_LOG, "skip %s %s " ZINT_FORMAT,
 633                          recordType, pr_fname, (zint) start_offset);
 634             *more = 1;
 635
 636             end_offset = stream->endf(stream, 0);
 637             if (end_offset)
 638                 stream->seekf(stream, end_offset);
 639
 640             return ZEBRA_OK;
 641         case RECCTRL_EXTRACT_OK:
 642             break;
 643         default:
 644             yaz_log (YLOG_WARN, "extract error: unknown error: %d", r);
 645             return ZEBRA_FAIL;
 646         }
 647         end_offset = stream->endf(stream, 0);
 648         if (end_offset)
 649             stream->seekf(stream, end_offset);
 650         else
 651             end_offset = stream->tellf(stream);
 652
 653         all_matches_add(&extractCtrl);
 654
 655         if (extractCtrl.match_criteria[0])
 656             match_criteria = extractCtrl.match_criteria;
 657     }
 658
 659     *more = 1;
 660     if (!sysno)
 661     {
 662         sysno = &sysno0;
 663
 664         if (match_criteria && *match_criteria) {
 665             matchStr = match_criteria;
 666         } else {
 667             if (zh->m_record_id && *zh->m_record_id) {
 668                 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
 669                                                zh->m_record_id);
 670                 if (!matchStr)
 671                 {
 672                     yaz_log (YLOG_LOG, "error %s %s " ZINT_FORMAT, recordType,
 673                              pr_fname, (zint) start_offset);
 674                     return ZEBRA_FAIL;
 675                 }
 676             }
 677         }
 678         if (matchStr)
 679         {
 680             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 681             char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
 682                                           matchStr);
 683             if (rinfo)
 684             {
 685                 assert(*rinfo == sizeof(*sysno));
 686                 memcpy (sysno, rinfo+1, sizeof(*sysno));
 687             }
 688         }
 689     }
 690     if (zebra_rec_keys_empty(zh->reg->keys))
 691     {
 692         /* the extraction process returned no information - the record
 693            is probably empty - unless flagShowRecords is in use */
 694         if (test_mode)
 695             return ZEBRA_OK;
 696     }
 697
 698     if (! *sysno)
 699     {
 700         /* new record */
 701         if (action == action_delete)
 702         {
 703             yaz_log (YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 704                          pr_fname, (zint) start_offset);
 705             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
 706             return ZEBRA_FAIL;
 707         }
 708         else if (action == action_replace)
 709         {
 710             yaz_log (YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 711                          pr_fname, (zint) start_offset);
 712             yaz_log (YLOG_WARN, "cannot update record above (seems new)");
 713             return ZEBRA_FAIL;
 714         }
 715         if (show_progress)
 716             yaz_log (YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
 717                      (zint) start_offset);
 718         rec = rec_new (zh->reg->records);
 719
 720         *sysno = rec->sysno;
 721
 722         recordAttr = rec_init_attr (zh->reg->zei, rec);
 723         if (extractCtrl.staticrank < 0)
 724         {
 725             yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0");
 726             extractCtrl.staticrank = 0;
 727         }
 728         recordAttr->staticrank = extractCtrl.staticrank;
 729
 730         if (matchStr)
 731         {
 732             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 733             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
 734                             sizeof(*sysno), sysno);
 735         }
 736
 737         extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 738         extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 739                                   recordAttr->staticrank);
 740         zh->records_inserted++;
 741     }
 742     else
 743     {
 744         /* record already exists */
 745         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 746         zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
 747         if (action == action_insert)
 748         {
 749             yaz_log (YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
 750                          recordType, pr_fname, (zint) start_offset);
 751             logRecord(zh);
 752             return ZEBRA_FAIL;
 753         }
 754
 755         rec = rec_get (zh->reg->records, *sysno);
 756         assert (rec);
 757
 758         recordAttr = rec_init_attr (zh->reg->zei, rec);
 759
 760         /* decrease total size */
 761         zebraExplain_recordBytesIncrement (zh->reg->zei,
 762                                            - recordAttr->recordSize);
 763
 764         zebra_rec_keys_set_buf(delkeys,
 765                                rec->info[recInfo_delKeys],
 766                                rec->size[recInfo_delKeys],
 767                                0);
 768         zebra_rec_keys_set_buf(sortKeys,
 769                                rec->info[recInfo_sortKeys],
 770                                rec->size[recInfo_sortKeys],
 771                                0);
 772
 773         extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
 774         extract_flush_record_keys(zh, *sysno, 0, delkeys,
 775                                   recordAttr->staticrank);
 776         if (action == action_delete)
 777         {
 778             /* record going to be deleted */
 779             if (zebra_rec_keys_empty(delkeys))
 780             {
 781                 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 782                         pr_fname, (zint) start_offset);
 783                 yaz_log(YLOG_WARN, "cannot delete file above, "
 784                         "storeKeys false (3)");
 785             }
 786             else
 787             {
 788                 if (show_progress)
 789                     yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 790                             pr_fname, (zint) start_offset);
 791                 zh->records_deleted++;
 792                 if (matchStr)
 793                 {
 794                     int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 795                     dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
 796                 }
 797                 rec_del (zh->reg->records, &rec);
 798             }
 799             rec_free(&rec);
 800             logRecord(zh);
 801             return ZEBRA_OK;
 802         }
 803         else
 804         {   /* update or special_update */
 805             if (show_progress)
 806                 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 807                         pr_fname, (zint) start_offset);
 808             recordAttr->staticrank = extractCtrl.staticrank;
 809             extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 810             extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 811                                       recordAttr->staticrank);
 812             zh->records_updated++;
 813         }
 814         zebra_rec_keys_close(delkeys);
 815         zebra_rec_keys_close(sortKeys);
 816     }
 817     /* update file type */
 818     xfree (rec->info[recInfo_fileType]);
 819     rec->info[recInfo_fileType] =
 820         rec_strdup (recordType, &rec->size[recInfo_fileType]);
 821
 822     /* update filename */
 823     xfree (rec->info[recInfo_filename]);
 824     rec->info[recInfo_filename] =
 825         rec_strdup (fname, &rec->size[recInfo_filename]);
 826
 827     /* update delete keys */
 828     xfree (rec->info[recInfo_delKeys]);
 829     if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
 830     {
 831         zebra_rec_keys_get_buf(zh->reg->keys,
 832                                &rec->info[recInfo_delKeys],
 833                                &rec->size[recInfo_delKeys]);
 834     }
 835     else
 836     {
 837         rec->info[recInfo_delKeys] = NULL;
 838         rec->size[recInfo_delKeys] = 0;
 839     }
 840     /* update sort keys */
 841     xfree (rec->info[recInfo_sortKeys]);
 842
 843     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 844                            &rec->info[recInfo_sortKeys],
 845                            &rec->size[recInfo_sortKeys]);
 846
 847     if (stream)
 848     {
 849         recordAttr->recordSize = end_offset - start_offset;
 850         zebraExplain_recordBytesIncrement(zh->reg->zei,
 851                                           recordAttr->recordSize);
 852     }
 853
 854     /* set run-number for this record */
 855     recordAttr->runNumber =
 856         zebraExplain_runNumberIncrement (zh->reg->zei, 0);
 857
 858     /* update store data */
 859     xfree (rec->info[recInfo_storeData]);
 860
 861     /* update store data */
 862     if (zh->store_data_buf)
 863     {
 864         rec->size[recInfo_storeData] = zh->store_data_size;
 865         rec->info[recInfo_storeData] = zh->store_data_buf;
 866         zh->store_data_buf = 0;
 867         recordAttr->recordSize = zh->store_data_size;
 868     }
 869     else if (zh->m_store_data)
 870     {
 871         off_t cur_offset = stream->tellf(stream);
 872
 873         rec->size[recInfo_storeData] = recordAttr->recordSize;
 874         rec->info[recInfo_storeData] = (char *)
 875             xmalloc (recordAttr->recordSize);
 876         stream->seekf(stream, start_offset);
 877         stream->readf(stream, rec->info[recInfo_storeData],
 878                       recordAttr->recordSize);
 879         stream->seekf(stream, cur_offset);
 880     }
 881     else
 882     {
 883         rec->info[recInfo_storeData] = NULL;
 884         rec->size[recInfo_storeData] = 0;
 885     }
 886     /* update database name */
 887     xfree (rec->info[recInfo_databaseName]);
 888     rec->info[recInfo_databaseName] =
 889         rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
 890
 891     /* update offset */
 892     recordAttr->recordOffset = start_offset;
 893
 894     /* commit this record */
 895     rec_put (zh->reg->records, &rec);
 896     logRecord(zh);
 897     return ZEBRA_OK;
 898 }
 899
 900 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
 901 {
 902     ZebraHandle zh = (ZebraHandle) handle;
 903     struct recExtractCtrl extractCtrl;
 904
 905     if (zebraExplain_curDatabase (zh->reg->zei,
 906                                   rec->info[recInfo_databaseName]))
 907     {
 908         abort();
 909         if (zebraExplain_newDatabase (zh->reg->zei,
 910                                       rec->info[recInfo_databaseName], 0))
 911             abort ();
 912     }
 913
 914     zebra_rec_keys_reset(zh->reg->keys);
 915     zebra_rec_keys_reset(zh->reg->sortKeys);
 916
 917     extractCtrl.init = extract_init;
 918     extractCtrl.tokenAdd = extract_token_add;
 919     extractCtrl.schemaAdd = extract_schema_add;
 920     extractCtrl.dh = zh->reg->dh;
 921
 922     init_extractCtrl(zh, &extractCtrl);
 923
 924     extractCtrl.flagShowRecords = 0;
 925     extractCtrl.match_criteria[0] = '\0';
 926     extractCtrl.staticrank = 0;
 927     extractCtrl.action = action_update;
 928
 929     extractCtrl.handle = handle;
 930     extractCtrl.first_record = 1;
 931
 932     extract_set_store_data_prepare(&extractCtrl);
 933
 934     if (n)
 935         grs_extract_tree(&extractCtrl, n);
 936
 937     if (rec->size[recInfo_delKeys])
 938     {
 939         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 940
 941         zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
 942
 943         zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
 944                                rec->size[recInfo_delKeys],
 945                                0);
 946         extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
 947         zebra_rec_keys_close(delkeys);
 948
 949         zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
 950                                rec->size[recInfo_sortKeys],
 951                                0);
 952
 953         extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
 954         zebra_rec_keys_close(sortkeys);
 955     }
 956     extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
 957     extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
 958
 959     xfree (rec->info[recInfo_delKeys]);
 960     zebra_rec_keys_get_buf(zh->reg->keys,
 961                            &rec->info[recInfo_delKeys],
 962                            &rec->size[recInfo_delKeys]);
 963
 964     xfree (rec->info[recInfo_sortKeys]);
 965     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 966                            &rec->info[recInfo_sortKeys],
 967                            &rec->size[recInfo_sortKeys]);
 968     return ZEBRA_OK;
 969 }
 970
 971 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
 972                           zebra_rec_keys_t reckeys,
 973                           int level)
 974 {
 975     if (zebra_rec_keys_rewind(reckeys))
 976     {
 977         size_t slen;
 978         const char *str;
 979         struct it_key key;
 980         NMEM nmem = nmem_create();
 981
 982         while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
 983         {
 984             char keystr[200]; /* room for zints to print */
 985             char *dst_term = 0;
 986             int ord = CAST_ZINT_TO_INT(key.mem[0]);
 987             int index_type, i;
 988             const char *string_index;
 989
 990             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
 991                                     0/* db */, &string_index);
 992             assert(index_type);
 993             zebra_term_untrans_iconv(zh, nmem, index_type,
 994                                      &dst_term, str);
 995             *keystr = '\0';
 996             for (i = 0; i<key.len; i++)
 997             {
 998                 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key.mem[i]);
 999             }
1000
1001             if (*str < CHR_BASE_CHAR)
1002             {
1003                 int i;
1004                 char dst_buf[200]; /* room for special chars */
1005
1006                 strcpy(dst_buf , "?");
1007
1008                 if (!strcmp(str, ""))
1009                     strcpy(dst_buf, "alwaysmatches");
1010                 if (!strcmp(str, FIRST_IN_FIELD_STR))
1011                     strcpy(dst_buf, "firstinfield");
1012                 else if (!strcmp(str, CHR_UNKNOWN))
1013                     strcpy(dst_buf, "unknown");
1014                 else if (!strcmp(str, CHR_SPACE))
1015                     strcpy(dst_buf, "space");
1016
1017                 for (i = 0; i<slen; i++)
1018                 {
1019                     sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
1020                 }
1021                 yaz_log(level, "%s%c %s %s", keystr, index_type,
1022                         string_index, dst_buf);
1023
1024             }
1025             else
1026                 yaz_log(level, "%s%c %s \"%s\"", keystr, index_type,
1027                         string_index, dst_term);
1028
1029             nmem_reset(nmem);
1030         }
1031         nmem_destroy(nmem);
1032     }
1033 }
1034
1035 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
1036                              zebra_rec_keys_t reckeys)
1037 {
1038     ZebraExplainInfo zei = zh->reg->zei;
1039     struct ord_stat {
1040         int no;
1041         int ord;
1042         struct ord_stat *next;
1043     };
1044
1045     if (zebra_rec_keys_rewind(reckeys))
1046     {
1047         struct ord_stat *ord_list = 0;
1048         struct ord_stat *p;
1049         size_t slen;
1050         const char *str;
1051         struct it_key key_in;
1052         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1053         {
1054             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1055
1056             for (p = ord_list; p ; p = p->next)
1057                 if (p->ord == ord)
1058                 {
1059                     p->no++;
1060                     break;
1061                 }
1062             if (!p)
1063             {
1064                 p = xmalloc(sizeof(*p));
1065                 p->no = 1;
1066                 p->ord = ord;
1067                 p->next = ord_list;
1068                 ord_list = p;
1069             }
1070         }
1071
1072         p = ord_list;
1073         while (p)
1074         {
1075             struct ord_stat *p1 = p;
1076
1077             if (is_insert)
1078                 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
1079             else
1080                 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1081             p = p->next;
1082             xfree(p1);
1083         }
1084     }
1085 }
1086
1087 void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd,
1088                                zebra_rec_keys_t reckeys,
1089                                zint staticrank)
1090 {
1091     ZebraExplainInfo zei = zh->reg->zei;
1092
1093     extract_rec_keys_adjust(zh, cmd, reckeys);
1094
1095     if (log_level_details)
1096     {
1097         yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1098                 sysno, cmd ? "insert" : "delete");
1099         extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1100     }
1101
1102     if (!zh->reg->key_block)
1103     {
1104         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1105         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1106         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1107         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1108     }
1109     zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1110
1111     if (zebra_rec_keys_rewind(reckeys))
1112     {
1113         size_t slen;
1114         const char *str;
1115         struct it_key key_in;
1116         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1117         {
1118             key_block_write(zh->reg->key_block, sysno,
1119                             &key_in, cmd, str, slen,
1120                             staticrank, zh->m_staticrank);
1121         }
1122     }
1123 }
1124
1125
1126 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
1127                                      zebra_rec_keys_t reckeys,
1128                                      zebra_snippets *snippets)
1129 {
1130     NMEM nmem = nmem_create();
1131     if (zebra_rec_keys_rewind(reckeys))
1132     {
1133         const char *str;
1134         size_t slen;
1135         struct it_key key;
1136         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1137         {
1138             char *dst_term = 0;
1139             int ord;
1140             zint seqno;
1141             int index_type;
1142
1143             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1144             seqno = key.mem[key.len-1];
1145             ord = CAST_ZINT_TO_INT(key.mem[0]);
1146
1147             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1148                                     0/* db */, 0 /* string_index */);
1149             assert(index_type);
1150             zebra_term_untrans_iconv(zh, nmem, index_type,
1151                                      &dst_term, str);
1152             zebra_snippets_append(snippets, seqno, ord, dst_term);
1153             nmem_reset(nmem);
1154         }
1155     }
1156     nmem_destroy(nmem);
1157     return ZEBRA_OK;
1158 }
1159
1160 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1161 {
1162     yaz_log(YLOG_LOG, "print_rec_keys");
1163     if (zebra_rec_keys_rewind(reckeys))
1164     {
1165         const char *str;
1166         size_t slen;
1167         struct it_key key;
1168         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1169         {
1170             char dst_buf[IT_MAX_WORD];
1171             zint seqno;
1172             int index_type;
1173             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1174             const char *db = 0;
1175             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1176
1177             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1178
1179             seqno = key.mem[key.len-1];
1180
1181             zebra_term_untrans(zh, index_type, dst_buf, str);
1182
1183             yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1184                     " term=%s", ord, seqno, dst_buf);
1185         }
1186     }
1187 }
1188
1189 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1190                                      const char *str, int length)
1191 {
1192     struct it_key key;
1193     ZebraHandle zh = p->extractCtrl->handle;
1194     ZebraExplainInfo zei = zh->reg->zei;
1195     int ch, i;
1196
1197     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1198     if (ch < 0)
1199         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1200
1201     i = 0;
1202     key.mem[i++] = ch;
1203     key.mem[i++] = p->record_id;
1204     key.mem[i++] = p->section_id;
1205
1206     if (zh->m_segment_indexing)
1207         key.mem[i++] = p->segment;
1208     key.mem[i++] = p->seqno;
1209     key.len = i;
1210
1211     zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1212 }
1213
1214 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1215 {
1216     struct it_key key;
1217     ZebraHandle zh = p->extractCtrl->handle;
1218     ZebraExplainInfo zei = zh->reg->zei;
1219     int ch;
1220     zinfo_index_category_t cat = zinfo_index_category_sort;
1221
1222     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1223     if (ch < 0)
1224         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1225     key.len = 2;
1226     key.mem[0] = ch;
1227     key.mem[1] = p->record_id;
1228
1229     zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1230 }
1231
1232 static void extract_add_staticrank_string(RecWord *p,
1233                                           const char *str, int length)
1234 {
1235     char valz[40];
1236     struct recExtractCtrl *ctrl = p->extractCtrl;
1237
1238     if (length > sizeof(valz)-1)
1239         length = sizeof(valz)-1;
1240
1241     memcpy(valz, str, length);
1242     valz[length] = '\0';
1243     ctrl->staticrank = atozint(valz);
1244 }
1245
1246 static void extract_add_string(RecWord *p, const char *string, int length)
1247 {
1248     ZebraHandle zh = p->extractCtrl->handle;
1249     assert (length > 0);
1250
1251     if (!p->index_name)
1252         return;
1253
1254     if (zebra_maps_is_index(zh->reg->zebra_maps, p->index_type))
1255     {
1256         extract_add_index_string(p, zinfo_index_category_index,
1257                                  string, length);
1258         if (zebra_maps_is_alwaysmatches(zh->reg->zebra_maps, p->index_type))
1259         {
1260             RecWord word;
1261             memcpy(&word, p, sizeof(word));
1262
1263             word.seqno = 1;
1264             extract_add_index_string(
1265                 &word, zinfo_index_category_alwaysmatches, "", 0);
1266         }
1267     }
1268     else if (zebra_maps_is_sort(zh->reg->zebra_maps, p->index_type))
1269     {
1270         extract_add_sort_string(p, string, length);
1271     }
1272     else if (zebra_maps_is_staticrank(zh->reg->zebra_maps, p->index_type))
1273     {
1274         extract_add_staticrank_string(p, string, length);
1275     }
1276 }
1277
1278 static void extract_add_incomplete_field(RecWord *p)
1279 {
1280     ZebraHandle zh = p->extractCtrl->handle;
1281     const char *b = p->term_buf;
1282     int remain = p->term_len;
1283     int first = 1;
1284     const char **map = 0;
1285
1286     if (remain > 0)
1287         map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1288
1289     while (map)
1290     {
1291         char buf[IT_MAX_WORD+1];
1292         int i, remain;
1293
1294         /* Skip spaces */
1295         while (map && *map && **map == *CHR_SPACE)
1296         {
1297             remain = p->term_len - (b - p->term_buf);
1298             if (remain > 0)
1299                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b,
1300                                        remain, 0);
1301             else
1302                 map = 0;
1303         }
1304         if (!map)
1305             break;
1306         i = 0;
1307         while (map && *map && **map != *CHR_SPACE)
1308         {
1309             const char *cp = *map;
1310
1311             while (i < IT_MAX_WORD && *cp)
1312                 buf[i++] = *(cp++);
1313             remain = p->term_len - (b - p->term_buf);
1314             if (remain > 0)
1315                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1316             else
1317                 map = 0;
1318         }
1319         if (!i)
1320             return;
1321
1322         if (first)
1323         {
1324             first = 0;
1325             if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type))
1326             {
1327                 /* first in field marker */
1328                 extract_add_string(p, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1329                 p->seqno++;
1330             }
1331         }
1332         extract_add_string (p, buf, i);
1333         p->seqno++;
1334     }
1335 }
1336
1337 static void extract_add_complete_field (RecWord *p)
1338 {
1339     ZebraHandle zh = p->extractCtrl->handle;
1340     const char *b = p->term_buf;
1341     char buf[IT_MAX_WORD+1];
1342     const char **map = 0;
1343     int i = 0, remain = p->term_len;
1344
1345     if (remain > 0)
1346         map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, remain, 1);
1347
1348     while (remain > 0 && i < IT_MAX_WORD)
1349     {
1350         while (map && *map && **map == *CHR_SPACE)
1351         {
1352             remain = p->term_len - (b - p->term_buf);
1353
1354             if (remain > 0)
1355             {
1356                 int first = i ? 0 : 1;  /* first position */
1357                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, first);
1358             }
1359             else
1360                 map = 0;
1361         }
1362         if (!map)
1363             break;
1364
1365         if (i && i < IT_MAX_WORD)
1366             buf[i++] = *CHR_SPACE;
1367         while (map && *map && **map != *CHR_SPACE)
1368         {
1369             const char *cp = *map;
1370
1371             if (**map == *CHR_CUT)
1372             {
1373                 i = 0;
1374             }
1375             else
1376             {
1377                 if (i >= IT_MAX_WORD)
1378                     break;
1379                 while (i < IT_MAX_WORD && *cp)
1380                     buf[i++] = *(cp++);
1381             }
1382             remain = p->term_len  - (b - p->term_buf);
1383             if (remain > 0)
1384             {
1385                 map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b,
1386                                         remain, 0);
1387             }
1388             else
1389                 map = 0;
1390         }
1391     }
1392     if (!i)
1393         return;
1394     extract_add_string (p, buf, i);
1395 }
1396
1397 static void extract_token_add(RecWord *p)
1398 {
1399     ZebraHandle zh = p->extractCtrl->handle;
1400     WRBUF wrbuf;
1401
1402     if (log_level_extract)
1403     {
1404         yaz_log(log_level_extract, "extract_token_add "
1405                 "type=%c index=%s seqno=" ZINT_FORMAT " s=%.*s",
1406                 p->index_type, p->index_name,
1407                 p->seqno, p->term_len, p->term_buf);
1408     }
1409     if ((wrbuf = zebra_replace(zh->reg->zebra_maps, p->index_type, 0,
1410                                p->term_buf, p->term_len)))
1411     {
1412         p->term_buf = wrbuf_buf(wrbuf);
1413         p->term_len = wrbuf_len(wrbuf);
1414     }
1415     if (zebra_maps_is_complete (zh->reg->zebra_maps, p->index_type))
1416         extract_add_complete_field (p);
1417     else
1418         extract_add_incomplete_field(p);
1419 }
1420
1421 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1422                                       void *buf, size_t sz)
1423 {
1424     ZebraHandle zh = (ZebraHandle) p->handle;
1425
1426     xfree(zh->store_data_buf);
1427     zh->store_data_buf = 0;
1428     zh->store_data_size = 0;
1429     if (buf && sz)
1430     {
1431         zh->store_data_buf = xmalloc(sz);
1432         zh->store_data_size = sz;
1433         memcpy(zh->store_data_buf, buf, sz);
1434     }
1435 }
1436
1437 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1438 {
1439     ZebraHandle zh = (ZebraHandle) p->handle;
1440     xfree(zh->store_data_buf);
1441     zh->store_data_buf = 0;
1442     zh->store_data_size = 0;
1443     p->setStoreData = extract_set_store_data_cb;
1444 }
1445
1446 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1447 {
1448     ZebraHandle zh = (ZebraHandle) p->handle;
1449     zebraExplain_addSchema (zh->reg->zei, oid);
1450 }
1451
1452 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1453                              int cmd, zebra_rec_keys_t reckeys)
1454 {
1455 #if 0
1456     yaz_log(YLOG_LOG, "extract_flush_sort_keys cmd=%d sysno=" ZINT_FORMAT,
1457             cmd, sysno);
1458     extract_rec_keys_log(zh, cmd, reckeys, YLOG_LOG);
1459 #endif
1460
1461     if (zebra_rec_keys_rewind(reckeys))
1462     {
1463         zebra_sort_index_t si = zh->reg->sort_index;
1464         size_t slen;
1465         const char *str;
1466         struct it_key key_in;
1467
1468         zebra_sort_sysno(si, sysno);
1469
1470         while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1471         {
1472             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1473
1474             zebra_sort_type(si, ord);
1475             if (cmd == 1)
1476                 zebra_sort_add(si, str, slen);
1477             else
1478                 zebra_sort_delete(si);
1479         }
1480     }
1481 }
1482
1483 /*
1484  * Local variables:
1485  * c-basic-offset: 4
1486  * indent-tabs-mode: nil
1487  * End:
1488  * vim: shiftwidth=4 tabstop=8 expandtab
1489  */
1490