index/extract.c

   1 /* $Id: extract.c,v 1.251 2007-03-13 13:46:11 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #endif
  29 #if HAVE_UNISTD_H
  30 #include <unistd.h>
  31 #endif
  32 #include <fcntl.h>
  33
  34 #include "index.h"
  35 #include "orddict.h"
  36 #include <direntz.h>
  37 #include <charmap.h>
  38
  39 static int log_level_extract = 0;
  40 static int log_level_details = 0;
  41 static int log_level_initialized = 0;
  42
  43 static void zebra_init_log_level(void)
  44 {
  45     if (!log_level_initialized)
  46     {
  47         log_level_initialized = 1;
  48
  49         log_level_extract = yaz_log_module_level("extract");
  50         log_level_details = yaz_log_module_level("indexdetails");
  51     }
  52 }
  53
  54 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
  55                                       int cmd, zebra_rec_keys_t reckeys,
  56                                       zint staticrank);
  57 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
  58                                     int cmd, zebra_rec_keys_t skp);
  59 static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid);
  60 static void extract_token_add (RecWord *p);
  61
  62 static void check_log_limit(ZebraHandle zh)
  63 {
  64     if (zh->records_processed + zh->records_skipped == zh->m_file_verbose_limit)
  65     {
  66         yaz_log(YLOG_LOG, "More than %d file log entries. Omitting rest",
  67                 zh->m_file_verbose_limit);
  68     }
  69 }
  70
  71 static void logRecord (ZebraHandle zh)
  72 {
  73     check_log_limit(zh);
  74     ++zh->records_processed;
  75     if (!(zh->records_processed % 1000))
  76     {
  77         yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
  78                 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
  79                 zh->records_processed, zh->records_inserted,
  80                 zh->records_updated, zh->records_deleted);
  81     }
  82 }
  83
  84 static void extract_add_index_string (RecWord *p,
  85                                       zinfo_index_category_t cat,
  86                                       const char *str, int length);
  87
  88 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
  89
  90 static void extract_init(struct recExtractCtrl *p, RecWord *w)
  91 {
  92     w->seqno = 1;
  93     w->index_name = "any";
  94     w->index_type = 'w';
  95     w->extractCtrl = p;
  96     w->record_id = 0;
  97     w->section_id = 0;
  98     w->segment = 0;
  99 }
 100
 101 static void searchRecordKey(ZebraHandle zh,
 102                             zebra_rec_keys_t reckeys,
 103                             const char *index_name,
 104                             const char **ws, int ws_length)
 105 {
 106     int i;
 107     int ch = -1;
 108     zinfo_index_category_t cat = zinfo_index_category_index;
 109
 110     for (i = 0; i<ws_length; i++)
 111         ws[i] = NULL;
 112
 113     if (ch < 0)
 114         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, '0', index_name);
 115     if (ch < 0)
 116         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'p', index_name);
 117     if (ch < 0)
 118         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'w', index_name);
 119
 120     if (ch < 0)
 121         return ;
 122
 123     if (zebra_rec_keys_rewind(reckeys))
 124     {
 125         zint startSeq = -1;
 126         const char *str;
 127         size_t slen;
 128         struct it_key key;
 129         zint seqno;
 130         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
 131         {
 132             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
 133
 134             seqno = key.mem[key.len-1];
 135
 136             if (key.mem[0] == ch)
 137             {
 138                 zint woff;
 139
 140                 if (startSeq == -1)
 141                     startSeq = seqno;
 142                 woff = seqno - startSeq;
 143                 if (woff >= 0 && woff < ws_length)
 144                     ws[woff] = str;
 145             }
 146         }
 147     }
 148 }
 149
 150 #define FILE_MATCH_BLANK "\t "
 151
 152 static char *get_match_from_spec(ZebraHandle zh,
 153                           zebra_rec_keys_t reckeys,
 154                           const char *fname, const char *spec)
 155 {
 156     static char dstBuf[2048];      /* static here ??? */
 157     char *dst = dstBuf;
 158     const char *s = spec;
 159
 160     while (1)
 161     {
 162         for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
 163             ;
 164         if (!*s)
 165             break;
 166         if (*s == '(')
 167         {
 168             const char *ws[32];
 169             char attset_str[64], attname_str[64];
 170             int i;
 171             int first = 1;
 172
 173             for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 174                 ;
 175             for (i = 0; *s && *s != ',' && *s != ')' &&
 176                      !strchr(FILE_MATCH_BLANK, *s); s++)
 177                 if (i+1 < sizeof(attset_str))
 178                     attset_str[i++] = *s;
 179             attset_str[i] = '\0';
 180
 181             for (; strchr(FILE_MATCH_BLANK, *s); s++)
 182                 ;
 183             if (*s != ',')
 184                 strcpy(attname_str, attset_str);
 185             else
 186             {
 187                 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 188                     ;
 189                 for (i = 0; *s && *s != ')' &&
 190                          !strchr(FILE_MATCH_BLANK, *s); s++)
 191                     if (i+1 < sizeof(attname_str))
 192                         attname_str[i++] = *s;
 193                 attname_str[i] = '\0';
 194             }
 195
 196             searchRecordKey (zh, reckeys, attname_str, ws, 32);
 197
 198             if (*s != ')')
 199             {
 200                 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
 201                       spec, zh->m_group ? zh->m_group : "none");
 202                 return NULL;
 203             }
 204             s++;
 205
 206             for (i = 0; i<32; i++)
 207                 if (ws[i])
 208                 {
 209                     if (first)
 210                     {
 211                         *dst++ = ' ';
 212                         first = 0;
 213                     }
 214                     strcpy (dst, ws[i]);
 215                     dst += strlen(ws[i]);
 216                 }
 217             if (first)
 218             {
 219                 yaz_log (YLOG_WARN, "Record didn't contain match"
 220                       " fields in (%s,%s)", attset_str, attname_str);
 221                 return NULL;
 222             }
 223         }
 224         else if (*s == '$')
 225         {
 226             int spec_len;
 227             char special[64];
 228             const char *spec_src = NULL;
 229             const char *s1 = ++s;
 230             while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
 231                 s1++;
 232
 233             spec_len = s1 - s;
 234             if (spec_len > sizeof(special)-1)
 235                 spec_len = sizeof(special)-1;
 236             memcpy (special, s, spec_len);
 237             special[spec_len] = '\0';
 238             s = s1;
 239
 240             if (!strcmp (special, "group"))
 241                 spec_src = zh->m_group;
 242             else if (!strcmp (special, "database"))
 243                 spec_src = zh->basenames[0];
 244             else if (!strcmp (special, "filename")) {
 245                 spec_src = fname;
 246             }
 247             else if (!strcmp (special, "type"))
 248                 spec_src = zh->m_record_type;
 249             else
 250                 spec_src = NULL;
 251             if (spec_src)
 252             {
 253                 strcpy (dst, spec_src);
 254                 dst += strlen (spec_src);
 255             }
 256         }
 257         else if (*s == '\"' || *s == '\'')
 258         {
 259             int stopMarker = *s++;
 260             char tmpString[64];
 261             int i = 0;
 262
 263             while (*s && *s != stopMarker)
 264             {
 265                 if (i+1 < sizeof(tmpString))
 266                     tmpString[i++] = *s++;
 267             }
 268             if (*s)
 269                 s++;
 270             tmpString[i] = '\0';
 271             strcpy (dst, tmpString);
 272             dst += strlen (tmpString);
 273         }
 274         else
 275         {
 276             yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
 277                   spec, zh->m_group ? zh->m_group : "none");
 278             return NULL;
 279         }
 280         *dst++ = 1;
 281     }
 282     if (dst == dstBuf)
 283     {
 284         yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
 285               fname, zh->m_group ? zh->m_group : "none");
 286         return NULL;
 287     }
 288     *dst = '\0';
 289     return dstBuf;
 290 }
 291
 292 struct recordLogInfo {
 293     const char *fname;
 294     int recordOffset;
 295     struct recordGroup *rGroup;
 296 };
 297
 298 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
 299 {
 300     int i;
 301     for (i = 0; i<256; i++)
 302     {
 303         if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
 304             ctrl->seqno[i] = 1;
 305         else
 306             ctrl->seqno[i] = 0;
 307     }
 308     ctrl->flagShowRecords = !zh->m_flag_rw;
 309 }
 310
 311 static void all_matches_add(struct recExtractCtrl *ctrl)
 312 {
 313     RecWord word;
 314     extract_init(ctrl, &word);
 315     word.index_name = "_ALLRECORDS";
 316     word.index_type = 'w';
 317     word.seqno = 1;
 318     extract_add_index_string (&word, zinfo_index_category_alwaysmatches,
 319                               "", 0);
 320 }
 321
 322 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
 323                                        struct ZebraRecStream *stream,
 324                                        int delete_flag,
 325                                        int test_mode,
 326                                        const char *recordType,
 327                                        zint *sysno,
 328                                        const char *match_criteria,
 329                                        const char *fname,
 330                                        int force_update,
 331                                        int allow_update,
 332                                        RecType recType,
 333                                        void *recTypeClientData);
 334
 335
 336 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
 337                              int deleteFlag)
 338 {
 339     ZEBRA_RES r = ZEBRA_OK;
 340     int i, fd;
 341     char gprefix[128];
 342     char ext[128];
 343     char ext_res[128];
 344     struct file_read_info *fi = 0;
 345     const char *original_record_type = 0;
 346     RecType recType;
 347     void *recTypeClientData;
 348     struct ZebraRecStream stream, *streamp;
 349
 350     zebra_init_log_level();
 351
 352     if (!zh->m_group || !*zh->m_group)
 353         *gprefix = '\0';
 354     else
 355         sprintf (gprefix, "%s.", zh->m_group);
 356
 357     yaz_log(log_level_extract, "zebra_extract_file %s", fname);
 358
 359     /* determine file extension */
 360     *ext = '\0';
 361     for (i = strlen(fname); --i >= 0; )
 362         if (fname[i] == '/')
 363             break;
 364         else if (fname[i] == '.')
 365         {
 366             strcpy (ext, fname+i+1);
 367             break;
 368         }
 369     /* determine file type - depending on extension */
 370     original_record_type = zh->m_record_type;
 371     if (!zh->m_record_type)
 372     {
 373         sprintf (ext_res, "%srecordType.%s", gprefix, ext);
 374         zh->m_record_type = res_get (zh->res, ext_res);
 375     }
 376     if (!zh->m_record_type)
 377     {
 378         check_log_limit(zh);
 379         if (zh->records_processed + zh->records_skipped
 380             < zh->m_file_verbose_limit)
 381             yaz_log (YLOG_LOG, "? %s", fname);
 382         zh->records_skipped++;
 383         return 0;
 384     }
 385     /* determine match criteria */
 386     if (!zh->m_record_id)
 387     {
 388         sprintf (ext_res, "%srecordId.%s", gprefix, ext);
 389         zh->m_record_id = res_get (zh->res, ext_res);
 390     }
 391
 392     if (!(recType =
 393           recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
 394                           &recTypeClientData)))
 395     {
 396         yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
 397         return ZEBRA_FAIL;
 398     }
 399
 400     switch(recType->version)
 401     {
 402     case 0:
 403         break;
 404     default:
 405         yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
 406     }
 407     if (sysno && deleteFlag)
 408     {
 409         streamp = 0;
 410         fi = 0;
 411     }
 412     else
 413     {
 414         char full_rep[1024];
 415
 416         if (zh->path_reg && !yaz_is_abspath (fname))
 417         {
 418             strcpy (full_rep, zh->path_reg);
 419             strcat (full_rep, "/");
 420             strcat (full_rep, fname);
 421         }
 422         else
 423             strcpy (full_rep, fname);
 424
 425         if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
 426         {
 427             yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
 428             zh->m_record_type = original_record_type;
 429             return ZEBRA_FAIL;
 430         }
 431         streamp = &stream;
 432         zebra_create_stream_fd(streamp, fd, 0);
 433     }
 434     r = zebra_extract_records_stream(zh, streamp,
 435                                      deleteFlag,
 436                                      0, /* tst_mode */
 437                                      zh->m_record_type,
 438                                      sysno,
 439                                      0, /*match_criteria */
 440                                      fname,
 441                                      1, /* force_update */
 442                                      1, /* allow_update */
 443                                      recType, recTypeClientData);
 444     if (streamp)
 445         stream.destroy(streamp);
 446     zh->m_record_type = original_record_type;
 447     return r;
 448 }
 449
 450 /*
 451   If sysno is provided, then it's used to identify the reocord.
 452   If not, and match_criteria is provided, then sysno is guessed
 453   If not, and a record is provided, then sysno is got from there
 454
 455  */
 456
 457 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
 458                                       const char *buf, size_t buf_size,
 459                                       int delete_flag,
 460                                       int test_mode,
 461                                       const char *recordType,
 462                                       zint *sysno,
 463                                       const char *match_criteria,
 464                                       const char *fname,
 465                                       int force_update,
 466                                       int allow_update)
 467 {
 468     struct ZebraRecStream stream;
 469     ZEBRA_RES res;
 470     void *clientData;
 471     RecType recType = 0;
 472
 473     if (recordType && *recordType)
 474     {
 475         yaz_log(log_level_extract,
 476                 "Record type explicitly specified: %s", recordType);
 477         recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
 478                                   &clientData);
 479     }
 480     else
 481     {
 482         if (!(zh->m_record_type))
 483         {
 484             yaz_log (YLOG_WARN, "No such record type defined");
 485             return ZEBRA_FAIL;
 486         }
 487         yaz_log(log_level_extract, "Get record type from rgroup: %s",
 488                 zh->m_record_type);
 489         recType = recType_byName (zh->reg->recTypes, zh->res,
 490                                   zh->m_record_type, &clientData);
 491         recordType = zh->m_record_type;
 492     }
 493
 494     if (!recType)
 495     {
 496         yaz_log (YLOG_WARN, "No such record type: %s", recordType);
 497         return ZEBRA_FAIL;
 498     }
 499
 500     zebra_create_stream_mem(&stream, buf, buf_size);
 501
 502     res = zebra_extract_records_stream(zh, &stream,
 503                                        delete_flag,
 504                                        test_mode,
 505                                        recordType,
 506                                        sysno,
 507                                        match_criteria,
 508                                        fname,
 509                                        force_update,
 510                                        allow_update,
 511                                        recType, clientData);
 512     stream.destroy(&stream);
 513     return res;
 514 }
 515
 516 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
 517                                        struct ZebraRecStream *stream,
 518                                        int delete_flag,
 519                                        int test_mode,
 520                                        const char *recordType,
 521                                        zint *sysno,
 522                                        const char *match_criteria,
 523                                        const char *fname,
 524                                        int force_update,
 525                                        int allow_update,
 526                                        RecType recType,
 527                                        void *recTypeClientData)
 528 {
 529     ZEBRA_RES res = ZEBRA_OK;
 530     while (1)
 531     {
 532         int more = 0;
 533         res = zebra_extract_record_stream(zh, stream,
 534                                           delete_flag,
 535                                           test_mode,
 536                                           recordType,
 537                                           sysno,
 538                                           match_criteria,
 539                                           fname,
 540                                           force_update,
 541                                           allow_update,
 542                                           recType, recTypeClientData, &more);
 543         if (!more)
 544         {
 545             res = ZEBRA_OK;
 546             break;
 547         }
 548         if (res != ZEBRA_OK)
 549             break;
 550         if (sysno)
 551             break;
 552     }
 553     return res;
 554 }
 555
 556
 557 ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
 558                                       struct ZebraRecStream *stream,
 559                                       int delete_flag,
 560                                       int test_mode,
 561                                       const char *recordType,
 562                                       zint *sysno,
 563                                       const char *match_criteria,
 564                                       const char *fname,
 565                                       int force_update,
 566                                       int allow_update,
 567                                       RecType recType,
 568                                       void *recTypeClientData,
 569                                       int *more)
 570
 571 {
 572     zint sysno0 = 0;
 573     RecordAttr *recordAttr;
 574     struct recExtractCtrl extractCtrl;
 575     int r;
 576     const char *matchStr = 0;
 577     Record rec;
 578     off_t start_offset = 0, end_offset = 0;
 579     const char *pr_fname = fname;  /* filename to print .. */
 580     int show_progress = zh->records_processed + zh->records_skipped
 581         < zh->m_file_verbose_limit ? 1:0;
 582
 583     zebra_init_log_level();
 584
 585     if (!pr_fname)
 586         pr_fname = "<no file>";  /* make it printable if file is omitted */
 587
 588     zebra_rec_keys_reset(zh->reg->keys);
 589     zebra_rec_keys_reset(zh->reg->sortKeys);
 590
 591     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
 592     {
 593         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
 594                                       zh->m_explain_database))
 595             return ZEBRA_FAIL;
 596     }
 597
 598     if (stream)
 599     {
 600         off_t null_offset = 0;
 601         extractCtrl.stream = stream;
 602
 603         start_offset = stream->tellf(stream);
 604
 605         extractCtrl.first_record = start_offset ? 0 : 1;
 606
 607         stream->endf(stream, &null_offset);;
 608
 609         extractCtrl.init = extract_init;
 610         extractCtrl.tokenAdd = extract_token_add;
 611         extractCtrl.schemaAdd = extract_schema_add;
 612         extractCtrl.dh = zh->reg->dh;
 613         extractCtrl.handle = zh;
 614         extractCtrl.match_criteria[0] = '\0';
 615         extractCtrl.staticrank = 0;
 616
 617         init_extractCtrl(zh, &extractCtrl);
 618
 619         extract_set_store_data_prepare(&extractCtrl);
 620
 621         r = (*recType->extract)(recTypeClientData, &extractCtrl);
 622
 623         switch (r)
 624         {
 625         case RECCTRL_EXTRACT_EOF:
 626             return ZEBRA_FAIL;
 627         case RECCTRL_EXTRACT_ERROR_GENERIC:
 628             /* error occured during extraction ... */
 629             yaz_log (YLOG_WARN, "extract error: generic");
 630             return ZEBRA_FAIL;
 631         case RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER:
 632             /* error occured during extraction ... */
 633             yaz_log (YLOG_WARN, "extract error: no such filter");
 634             return ZEBRA_FAIL;
 635         case RECCTRL_EXTRACT_SKIP:
 636             if (show_progress)
 637                 yaz_log (YLOG_LOG, "skip %s %s " ZINT_FORMAT,
 638                          recordType, pr_fname, (zint) start_offset);
 639             *more = 1;
 640
 641             end_offset = stream->endf(stream, 0);
 642             if (end_offset)
 643                 stream->seekf(stream, end_offset);
 644
 645             return ZEBRA_OK;
 646         case RECCTRL_EXTRACT_OK:
 647             break;
 648         default:
 649             yaz_log (YLOG_WARN, "extract error: unknown error: %d", r);
 650             return ZEBRA_FAIL;
 651         }
 652         end_offset = stream->endf(stream, 0);
 653         if (end_offset)
 654             stream->seekf(stream, end_offset);
 655         else
 656             end_offset = stream->tellf(stream);
 657
 658         all_matches_add(&extractCtrl);
 659
 660         if (extractCtrl.match_criteria[0])
 661             match_criteria = extractCtrl.match_criteria;
 662     }
 663
 664     *more = 1;
 665     if (!sysno)
 666     {
 667         sysno = &sysno0;
 668
 669         if (match_criteria && *match_criteria) {
 670             matchStr = match_criteria;
 671         } else {
 672             if (zh->m_record_id && *zh->m_record_id) {
 673                 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
 674                                                zh->m_record_id);
 675                 if (!matchStr)
 676                 {
 677                     yaz_log (YLOG_LOG, "error %s %s " ZINT_FORMAT, recordType,
 678                              pr_fname, (zint) start_offset);
 679                     return ZEBRA_FAIL;
 680                 }
 681             }
 682         }
 683         if (matchStr)
 684         {
 685             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 686             char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
 687                                           matchStr);
 688             if (rinfo)
 689             {
 690                 assert(*rinfo == sizeof(*sysno));
 691                 memcpy (sysno, rinfo+1, sizeof(*sysno));
 692             }
 693         }
 694     }
 695     if (zebra_rec_keys_empty(zh->reg->keys))
 696     {
 697         /* the extraction process returned no information - the record
 698            is probably empty - unless flagShowRecords is in use */
 699         if (test_mode)
 700             return ZEBRA_OK;
 701     }
 702
 703     if (! *sysno)
 704     {
 705         /* new record */
 706         if (delete_flag)
 707         {
 708             yaz_log (YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 709                          pr_fname, (zint) start_offset);
 710             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
 711             return ZEBRA_FAIL;
 712         }
 713         else if (!force_update)
 714         {
 715             yaz_log (YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 716                          pr_fname, (zint) start_offset);
 717             yaz_log (YLOG_WARN, "cannot update record above (seems new)");
 718             return ZEBRA_FAIL;
 719         }
 720         if (show_progress)
 721             yaz_log (YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
 722                      (zint) start_offset);
 723         rec = rec_new (zh->reg->records);
 724
 725         *sysno = rec->sysno;
 726
 727         recordAttr = rec_init_attr (zh->reg->zei, rec);
 728         if (extractCtrl.staticrank < 0)
 729         {
 730             yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0");
 731             extractCtrl.staticrank = 0;
 732         }
 733         recordAttr->staticrank = extractCtrl.staticrank;
 734
 735         if (matchStr)
 736         {
 737             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 738             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
 739                             sizeof(*sysno), sysno);
 740         }
 741
 742         extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 743         extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 744                                   recordAttr->staticrank);
 745         zh->records_inserted++;
 746     }
 747     else
 748     {
 749         /* record already exists */
 750         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 751         zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
 752         if (!allow_update)
 753         {
 754             yaz_log (YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
 755                          recordType, pr_fname, (zint) start_offset);
 756             logRecord(zh);
 757             return ZEBRA_FAIL;
 758         }
 759
 760         rec = rec_get (zh->reg->records, *sysno);
 761         assert (rec);
 762
 763         recordAttr = rec_init_attr (zh->reg->zei, rec);
 764
 765         /* decrease total size */
 766         zebraExplain_recordBytesIncrement (zh->reg->zei,
 767                                            - recordAttr->recordSize);
 768
 769         zebra_rec_keys_set_buf(delkeys,
 770                                rec->info[recInfo_delKeys],
 771                                rec->size[recInfo_delKeys],
 772                                0);
 773         zebra_rec_keys_set_buf(sortKeys,
 774                                rec->info[recInfo_sortKeys],
 775                                rec->size[recInfo_sortKeys],
 776                                0);
 777
 778         extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
 779         extract_flush_record_keys(zh, *sysno, 0, delkeys,
 780                                   recordAttr->staticrank);
 781         if (delete_flag)
 782         {
 783             /* record going to be deleted */
 784             if (zebra_rec_keys_empty(delkeys))
 785             {
 786                 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 787                         pr_fname, (zint) start_offset);
 788                 yaz_log(YLOG_WARN, "cannot delete file above, "
 789                         "storeKeys false (3)");
 790             }
 791             else
 792             {
 793                 if (show_progress)
 794                     yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 795                             pr_fname, (zint) start_offset);
 796                 zh->records_deleted++;
 797                 if (matchStr)
 798                 {
 799                     int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 800                     dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
 801                 }
 802                 rec_del (zh->reg->records, &rec);
 803             }
 804             rec_free(&rec);
 805             logRecord(zh);
 806             return ZEBRA_OK;
 807         }
 808         else
 809         {
 810             if (show_progress)
 811                 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 812                         pr_fname, (zint) start_offset);
 813             recordAttr->staticrank = extractCtrl.staticrank;
 814             extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 815             extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 816                                       recordAttr->staticrank);
 817             zh->records_updated++;
 818         }
 819         zebra_rec_keys_close(delkeys);
 820         zebra_rec_keys_close(sortKeys);
 821     }
 822     /* update file type */
 823     xfree (rec->info[recInfo_fileType]);
 824     rec->info[recInfo_fileType] =
 825         rec_strdup (recordType, &rec->size[recInfo_fileType]);
 826
 827     /* update filename */
 828     xfree (rec->info[recInfo_filename]);
 829     rec->info[recInfo_filename] =
 830         rec_strdup (fname, &rec->size[recInfo_filename]);
 831
 832     /* update delete keys */
 833     xfree (rec->info[recInfo_delKeys]);
 834     if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
 835     {
 836         zebra_rec_keys_get_buf(zh->reg->keys,
 837                                &rec->info[recInfo_delKeys],
 838                                &rec->size[recInfo_delKeys]);
 839     }
 840     else
 841     {
 842         rec->info[recInfo_delKeys] = NULL;
 843         rec->size[recInfo_delKeys] = 0;
 844     }
 845     /* update sort keys */
 846     xfree (rec->info[recInfo_sortKeys]);
 847
 848     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 849                            &rec->info[recInfo_sortKeys],
 850                            &rec->size[recInfo_sortKeys]);
 851
 852     if (stream)
 853     {
 854         recordAttr->recordSize = end_offset - start_offset;
 855         zebraExplain_recordBytesIncrement(zh->reg->zei,
 856                                           recordAttr->recordSize);
 857     }
 858
 859     /* set run-number for this record */
 860     recordAttr->runNumber =
 861         zebraExplain_runNumberIncrement (zh->reg->zei, 0);
 862
 863     /* update store data */
 864     xfree (rec->info[recInfo_storeData]);
 865
 866     /* update store data */
 867     if (zh->store_data_buf)
 868     {
 869         rec->size[recInfo_storeData] = zh->store_data_size;
 870         rec->info[recInfo_storeData] = zh->store_data_buf;
 871         zh->store_data_buf = 0;
 872         recordAttr->recordSize = zh->store_data_size;
 873     }
 874     else if (zh->m_store_data)
 875     {
 876         off_t cur_offset = stream->tellf(stream);
 877
 878         rec->size[recInfo_storeData] = recordAttr->recordSize;
 879         rec->info[recInfo_storeData] = (char *)
 880             xmalloc (recordAttr->recordSize);
 881         stream->seekf(stream, start_offset);
 882         stream->readf(stream, rec->info[recInfo_storeData],
 883                       recordAttr->recordSize);
 884         stream->seekf(stream, cur_offset);
 885     }
 886     else
 887     {
 888         rec->info[recInfo_storeData] = NULL;
 889         rec->size[recInfo_storeData] = 0;
 890     }
 891     /* update database name */
 892     xfree (rec->info[recInfo_databaseName]);
 893     rec->info[recInfo_databaseName] =
 894         rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
 895
 896     /* update offset */
 897     recordAttr->recordOffset = start_offset;
 898
 899     /* commit this record */
 900     rec_put (zh->reg->records, &rec);
 901     logRecord(zh);
 902     return ZEBRA_OK;
 903 }
 904
 905 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
 906 {
 907     ZebraHandle zh = (ZebraHandle) handle;
 908     struct recExtractCtrl extractCtrl;
 909
 910     if (zebraExplain_curDatabase (zh->reg->zei,
 911                                   rec->info[recInfo_databaseName]))
 912     {
 913         abort();
 914         if (zebraExplain_newDatabase (zh->reg->zei,
 915                                       rec->info[recInfo_databaseName], 0))
 916             abort ();
 917     }
 918
 919     zebra_rec_keys_reset(zh->reg->keys);
 920     zebra_rec_keys_reset(zh->reg->sortKeys);
 921
 922     extractCtrl.init = extract_init;
 923     extractCtrl.tokenAdd = extract_token_add;
 924     extractCtrl.schemaAdd = extract_schema_add;
 925     extractCtrl.dh = zh->reg->dh;
 926
 927     init_extractCtrl(zh, &extractCtrl);
 928
 929     extractCtrl.flagShowRecords = 0;
 930     extractCtrl.match_criteria[0] = '\0';
 931     extractCtrl.staticrank = 0;
 932     extractCtrl.handle = handle;
 933     extractCtrl.first_record = 1;
 934
 935     extract_set_store_data_prepare(&extractCtrl);
 936
 937     if (n)
 938         grs_extract_tree(&extractCtrl, n);
 939
 940     if (rec->size[recInfo_delKeys])
 941     {
 942         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 943
 944         zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
 945
 946         zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
 947                                rec->size[recInfo_delKeys],
 948                                0);
 949         extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
 950         zebra_rec_keys_close(delkeys);
 951
 952         zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
 953                                rec->size[recInfo_sortKeys],
 954                                0);
 955
 956         extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
 957         zebra_rec_keys_close(sortkeys);
 958     }
 959     extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
 960     extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
 961
 962     xfree (rec->info[recInfo_delKeys]);
 963     zebra_rec_keys_get_buf(zh->reg->keys,
 964                            &rec->info[recInfo_delKeys],
 965                            &rec->size[recInfo_delKeys]);
 966
 967     xfree (rec->info[recInfo_sortKeys]);
 968     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 969                            &rec->info[recInfo_sortKeys],
 970                            &rec->size[recInfo_sortKeys]);
 971     return ZEBRA_OK;
 972 }
 973
 974 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
 975                           zebra_rec_keys_t reckeys,
 976                           int level)
 977 {
 978     if (zebra_rec_keys_rewind(reckeys))
 979     {
 980         size_t slen;
 981         const char *str;
 982         struct it_key key;
 983         NMEM nmem = nmem_create();
 984
 985         while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
 986         {
 987             char keystr[200]; /* room for zints to print */
 988             char *dst_term = 0;
 989             int ord = CAST_ZINT_TO_INT(key.mem[0]);
 990             int index_type, i;
 991             const char *string_index;
 992
 993             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
 994                                     0/* db */, &string_index);
 995             assert(index_type);
 996             zebra_term_untrans_iconv(zh, nmem, index_type,
 997                                      &dst_term, str);
 998             *keystr = '\0';
 999             for (i = 0; i<key.len; i++)
1000             {
1001                 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key.mem[i]);
1002             }
1003
1004             if (*str < CHR_BASE_CHAR)
1005             {
1006                 int i;
1007                 char dst_buf[200]; /* room for special chars */
1008
1009                 strcpy(dst_buf , "?");
1010
1011                 if (!strcmp(str, ""))
1012                     strcpy(dst_buf, "alwaysmatches");
1013                 if (!strcmp(str, FIRST_IN_FIELD_STR))
1014                     strcpy(dst_buf, "firstinfield");
1015                 else if (!strcmp(str, CHR_UNKNOWN))
1016                     strcpy(dst_buf, "unknown");
1017                 else if (!strcmp(str, CHR_SPACE))
1018                     strcpy(dst_buf, "space");
1019
1020                 for (i = 0; i<slen; i++)
1021                 {
1022                     sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
1023                 }
1024                 yaz_log(level, "%s%c %s %s", keystr, index_type,
1025                         string_index, dst_buf);
1026
1027             }
1028             else
1029                 yaz_log(level, "%s%c %s \"%s\"", keystr, index_type,
1030                         string_index, dst_term);
1031
1032             nmem_reset(nmem);
1033         }
1034         nmem_destroy(nmem);
1035     }
1036 }
1037
1038 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
1039                              zebra_rec_keys_t reckeys)
1040 {
1041     ZebraExplainInfo zei = zh->reg->zei;
1042     struct ord_stat {
1043         int no;
1044         int ord;
1045         struct ord_stat *next;
1046     };
1047
1048     if (zebra_rec_keys_rewind(reckeys))
1049     {
1050         struct ord_stat *ord_list = 0;
1051         struct ord_stat *p;
1052         size_t slen;
1053         const char *str;
1054         struct it_key key_in;
1055         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1056         {
1057             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1058
1059             for (p = ord_list; p ; p = p->next)
1060                 if (p->ord == ord)
1061                 {
1062                     p->no++;
1063                     break;
1064                 }
1065             if (!p)
1066             {
1067                 p = xmalloc(sizeof(*p));
1068                 p->no = 1;
1069                 p->ord = ord;
1070                 p->next = ord_list;
1071                 ord_list = p;
1072             }
1073         }
1074
1075         p = ord_list;
1076         while (p)
1077         {
1078             struct ord_stat *p1 = p;
1079
1080             if (is_insert)
1081                 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
1082             else
1083                 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1084             p = p->next;
1085             xfree(p1);
1086         }
1087     }
1088 }
1089
1090 void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd,
1091                                zebra_rec_keys_t reckeys,
1092                                zint staticrank)
1093 {
1094     ZebraExplainInfo zei = zh->reg->zei;
1095
1096     extract_rec_keys_adjust(zh, cmd, reckeys);
1097
1098     if (log_level_details)
1099     {
1100         yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1101                 sysno, cmd ? "insert" : "delete");
1102         extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1103     }
1104
1105     if (!zh->reg->key_block)
1106     {
1107         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1108         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1109         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1110         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1111     }
1112     zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1113
1114     if (zebra_rec_keys_rewind(reckeys))
1115     {
1116         size_t slen;
1117         const char *str;
1118         struct it_key key_in;
1119         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1120         {
1121             key_block_write(zh->reg->key_block, sysno,
1122                             &key_in, cmd, str, slen,
1123                             staticrank, zh->m_staticrank);
1124         }
1125     }
1126 }
1127
1128
1129 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
1130                                      zebra_rec_keys_t reckeys,
1131                                      zebra_snippets *snippets)
1132 {
1133     NMEM nmem = nmem_create();
1134     if (zebra_rec_keys_rewind(reckeys))
1135     {
1136         const char *str;
1137         size_t slen;
1138         struct it_key key;
1139         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1140         {
1141             char *dst_term = 0;
1142             int ord;
1143             zint seqno;
1144             int index_type;
1145
1146             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1147             seqno = key.mem[key.len-1];
1148             ord = CAST_ZINT_TO_INT(key.mem[0]);
1149
1150             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1151                                     0/* db */, 0 /* string_index */);
1152             assert(index_type);
1153             zebra_term_untrans_iconv(zh, nmem, index_type,
1154                                      &dst_term, str);
1155             zebra_snippets_append(snippets, seqno, ord, dst_term);
1156             nmem_reset(nmem);
1157         }
1158     }
1159     nmem_destroy(nmem);
1160     return ZEBRA_OK;
1161 }
1162
1163 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1164 {
1165     yaz_log(YLOG_LOG, "print_rec_keys");
1166     if (zebra_rec_keys_rewind(reckeys))
1167     {
1168         const char *str;
1169         size_t slen;
1170         struct it_key key;
1171         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1172         {
1173             char dst_buf[IT_MAX_WORD];
1174             zint seqno;
1175             int index_type;
1176             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1177             const char *db = 0;
1178             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1179
1180             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1181
1182             seqno = key.mem[key.len-1];
1183
1184             zebra_term_untrans(zh, index_type, dst_buf, str);
1185
1186             yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1187                     " term=%s", ord, seqno, dst_buf);
1188         }
1189     }
1190 }
1191
1192 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1193                                      const char *str, int length)
1194 {
1195     struct it_key key;
1196     ZebraHandle zh = p->extractCtrl->handle;
1197     ZebraExplainInfo zei = zh->reg->zei;
1198     int ch, i;
1199
1200     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1201     if (ch < 0)
1202         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1203
1204     i = 0;
1205     key.mem[i++] = ch;
1206     key.mem[i++] = p->record_id;
1207     key.mem[i++] = p->section_id;
1208
1209     if (zh->m_segment_indexing)
1210         key.mem[i++] = p->segment;
1211     key.mem[i++] = p->seqno;
1212     key.len = i;
1213
1214     zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1215 }
1216
1217 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1218 {
1219     struct it_key key;
1220     ZebraHandle zh = p->extractCtrl->handle;
1221     ZebraExplainInfo zei = zh->reg->zei;
1222     int ch;
1223     zinfo_index_category_t cat = zinfo_index_category_sort;
1224
1225     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1226     if (ch < 0)
1227         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1228     key.len = 2;
1229     key.mem[0] = ch;
1230     key.mem[1] = p->record_id;
1231
1232     zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1233 }
1234
1235 static void extract_add_staticrank_string(RecWord *p,
1236                                           const char *str, int length)
1237 {
1238     char valz[40];
1239     struct recExtractCtrl *ctrl = p->extractCtrl;
1240
1241     if (length > sizeof(valz)-1)
1242         length = sizeof(valz)-1;
1243
1244     memcpy(valz, str, length);
1245     valz[length] = '\0';
1246     ctrl->staticrank = atozint(valz);
1247 }
1248
1249 static void extract_add_string(RecWord *p, const char *string, int length)
1250 {
1251     ZebraHandle zh = p->extractCtrl->handle;
1252     assert (length > 0);
1253
1254     if (!p->index_name)
1255         return;
1256
1257     if (zebra_maps_is_index(zh->reg->zebra_maps, p->index_type))
1258     {
1259         extract_add_index_string(p, zinfo_index_category_index,
1260                                  string, length);
1261         if (zebra_maps_is_alwaysmatches(zh->reg->zebra_maps, p->index_type))
1262         {
1263             RecWord word;
1264             memcpy(&word, p, sizeof(word));
1265
1266             word.seqno = 1;
1267             extract_add_index_string(
1268                 &word, zinfo_index_category_alwaysmatches, "", 0);
1269         }
1270     }
1271     else if (zebra_maps_is_sort(zh->reg->zebra_maps, p->index_type))
1272     {
1273         extract_add_sort_string(p, string, length);
1274     }
1275     else if (zebra_maps_is_staticrank(zh->reg->zebra_maps, p->index_type))
1276     {
1277         extract_add_staticrank_string(p, string, length);
1278     }
1279 }
1280
1281 static void extract_add_incomplete_field(RecWord *p)
1282 {
1283     ZebraHandle zh = p->extractCtrl->handle;
1284     const char *b = p->term_buf;
1285     int remain = p->term_len;
1286     int first = 1;
1287     const char **map = 0;
1288
1289     if (remain > 0)
1290         map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1291
1292     while (map)
1293     {
1294         char buf[IT_MAX_WORD+1];
1295         int i, remain;
1296
1297         /* Skip spaces */
1298         while (map && *map && **map == *CHR_SPACE)
1299         {
1300             remain = p->term_len - (b - p->term_buf);
1301             if (remain > 0)
1302                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b,
1303                                        remain, 0);
1304             else
1305                 map = 0;
1306         }
1307         if (!map)
1308             break;
1309         i = 0;
1310         while (map && *map && **map != *CHR_SPACE)
1311         {
1312             const char *cp = *map;
1313
1314             while (i < IT_MAX_WORD && *cp)
1315                 buf[i++] = *(cp++);
1316             remain = p->term_len - (b - p->term_buf);
1317             if (remain > 0)
1318                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1319             else
1320                 map = 0;
1321         }
1322         if (!i)
1323             return;
1324
1325         if (first)
1326         {
1327             first = 0;
1328             if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type))
1329             {
1330                 /* first in field marker */
1331                 extract_add_string(p, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1332                 p->seqno++;
1333             }
1334         }
1335         extract_add_string (p, buf, i);
1336         p->seqno++;
1337     }
1338 }
1339
1340 static void extract_add_complete_field (RecWord *p)
1341 {
1342     ZebraHandle zh = p->extractCtrl->handle;
1343     const char *b = p->term_buf;
1344     char buf[IT_MAX_WORD+1];
1345     const char **map = 0;
1346     int i = 0, remain = p->term_len;
1347
1348     if (remain > 0)
1349         map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, remain, 1);
1350
1351     while (remain > 0 && i < IT_MAX_WORD)
1352     {
1353         while (map && *map && **map == *CHR_SPACE)
1354         {
1355             remain = p->term_len - (b - p->term_buf);
1356
1357             if (remain > 0)
1358             {
1359                 int first = i ? 0 : 1;  /* first position */
1360                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, first);
1361             }
1362             else
1363                 map = 0;
1364         }
1365         if (!map)
1366             break;
1367
1368         if (i && i < IT_MAX_WORD)
1369             buf[i++] = *CHR_SPACE;
1370         while (map && *map && **map != *CHR_SPACE)
1371         {
1372             const char *cp = *map;
1373
1374             if (**map == *CHR_CUT)
1375             {
1376                 i = 0;
1377             }
1378             else
1379             {
1380                 if (i >= IT_MAX_WORD)
1381                     break;
1382                 while (i < IT_MAX_WORD && *cp)
1383                     buf[i++] = *(cp++);
1384             }
1385             remain = p->term_len  - (b - p->term_buf);
1386             if (remain > 0)
1387             {
1388                 map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b,
1389                                         remain, 0);
1390             }
1391             else
1392                 map = 0;
1393         }
1394     }
1395     if (!i)
1396         return;
1397     extract_add_string (p, buf, i);
1398 }
1399
1400 static void extract_token_add(RecWord *p)
1401 {
1402     ZebraHandle zh = p->extractCtrl->handle;
1403     WRBUF wrbuf;
1404
1405     if (log_level_extract)
1406     {
1407         yaz_log(log_level_extract, "extract_token_add "
1408                 "type=%c index=%s seqno=" ZINT_FORMAT " s=%.*s",
1409                 p->index_type, p->index_name,
1410                 p->seqno, p->term_len, p->term_buf);
1411     }
1412     if ((wrbuf = zebra_replace(zh->reg->zebra_maps, p->index_type, 0,
1413                                p->term_buf, p->term_len)))
1414     {
1415         p->term_buf = wrbuf_buf(wrbuf);
1416         p->term_len = wrbuf_len(wrbuf);
1417     }
1418     if (zebra_maps_is_complete (zh->reg->zebra_maps, p->index_type))
1419         extract_add_complete_field (p);
1420     else
1421         extract_add_incomplete_field(p);
1422 }
1423
1424 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1425                                       void *buf, size_t sz)
1426 {
1427     ZebraHandle zh = (ZebraHandle) p->handle;
1428
1429     xfree(zh->store_data_buf);
1430     zh->store_data_buf = 0;
1431     zh->store_data_size = 0;
1432     if (buf && sz)
1433     {
1434         zh->store_data_buf = xmalloc(sz);
1435         zh->store_data_size = sz;
1436         memcpy(zh->store_data_buf, buf, sz);
1437     }
1438 }
1439
1440 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1441 {
1442     ZebraHandle zh = (ZebraHandle) p->handle;
1443     xfree(zh->store_data_buf);
1444     zh->store_data_buf = 0;
1445     zh->store_data_size = 0;
1446     p->setStoreData = extract_set_store_data_cb;
1447 }
1448
1449 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1450 {
1451     ZebraHandle zh = (ZebraHandle) p->handle;
1452     zebraExplain_addSchema (zh->reg->zei, oid);
1453 }
1454
1455 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1456                              int cmd, zebra_rec_keys_t reckeys)
1457 {
1458 #if 0
1459     yaz_log(YLOG_LOG, "extract_flush_sort_keys cmd=%d sysno=" ZINT_FORMAT,
1460             cmd, sysno);
1461     extract_rec_keys_log(zh, cmd, reckeys, YLOG_LOG);
1462 #endif
1463
1464     if (zebra_rec_keys_rewind(reckeys))
1465     {
1466         zebra_sort_index_t si = zh->reg->sort_index;
1467         size_t slen;
1468         const char *str;
1469         struct it_key key_in;
1470
1471         zebra_sort_sysno(si, sysno);
1472
1473         while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1474         {
1475             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1476
1477             zebra_sort_type(si, ord);
1478             if (cmd == 1)
1479                 zebra_sort_add(si, str, slen);
1480             else
1481                 zebra_sort_delete(si);
1482         }
1483     }
1484 }
1485
1486 /*
1487  * Local variables:
1488  * c-basic-offset: 4
1489  * indent-tabs-mode: nil
1490  * End:
1491  * vim: shiftwidth=4 tabstop=8 expandtab
1492  */
1493