index/extract.c

   1 /* $Id: extract.c,v 1.250 2007-03-01 10:35:46 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #endif
  29 #if HAVE_UNISTD_H
  30 #include <unistd.h>
  31 #endif
  32 #include <fcntl.h>
  33
  34 #include "index.h"
  35 #include "orddict.h"
  36 #include <direntz.h>
  37 #include <charmap.h>
  38
  39 static int log_level_extract = 0;
  40 static int log_level_details = 0;
  41 static int log_level_initialized = 0;
  42
  43 static void zebra_init_log_level(void)
  44 {
  45     if (!log_level_initialized)
  46     {
  47         log_level_initialized = 1;
  48
  49         log_level_extract = yaz_log_module_level("extract");
  50         log_level_details = yaz_log_module_level("indexdetails");
  51     }
  52 }
  53
  54 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
  55                                       int cmd, zebra_rec_keys_t reckeys,
  56                                       zint staticrank);
  57 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
  58                                     int cmd, zebra_rec_keys_t skp);
  59 static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid);
  60 static void extract_token_add (RecWord *p);
  61
  62 static void check_log_limit(ZebraHandle zh)
  63 {
  64     if (zh->records_processed + zh->records_skipped == zh->m_file_verbose_limit)
  65     {
  66         yaz_log(YLOG_LOG, "More than %d file log entries. Omitting rest",
  67                 zh->m_file_verbose_limit);
  68     }
  69 }
  70
  71 static void logRecord (ZebraHandle zh)
  72 {
  73     check_log_limit(zh);
  74     ++zh->records_processed;
  75     if (!(zh->records_processed % 1000))
  76     {
  77         yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
  78                 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
  79                 zh->records_processed, zh->records_inserted,
  80                 zh->records_updated, zh->records_deleted);
  81     }
  82 }
  83
  84 static void extract_add_index_string (RecWord *p,
  85                                       zinfo_index_category_t cat,
  86                                       const char *str, int length);
  87
  88 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
  89
  90 static void extract_init(struct recExtractCtrl *p, RecWord *w)
  91 {
  92     w->seqno = 1;
  93     w->index_name = "any";
  94     w->index_type = 'w';
  95     w->extractCtrl = p;
  96     w->record_id = 0;
  97     w->section_id = 0;
  98     w->segment = 0;
  99 }
 100
 101 static void searchRecordKey(ZebraHandle zh,
 102                             zebra_rec_keys_t reckeys,
 103                             const char *index_name,
 104                             const char **ws, int ws_length)
 105 {
 106     int i;
 107     int ch = -1;
 108     zinfo_index_category_t cat = zinfo_index_category_index;
 109
 110     for (i = 0; i<ws_length; i++)
 111         ws[i] = NULL;
 112
 113     if (ch < 0)
 114         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, '0', index_name);
 115     if (ch < 0)
 116         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'p', index_name);
 117     if (ch < 0)
 118         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'w', index_name);
 119
 120     if (ch < 0)
 121         return ;
 122
 123     if (zebra_rec_keys_rewind(reckeys))
 124     {
 125         zint startSeq = -1;
 126         const char *str;
 127         size_t slen;
 128         struct it_key key;
 129         zint seqno;
 130         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
 131         {
 132             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
 133
 134             seqno = key.mem[key.len-1];
 135
 136             if (key.mem[0] == ch)
 137             {
 138                 zint woff;
 139
 140                 if (startSeq == -1)
 141                     startSeq = seqno;
 142                 woff = seqno - startSeq;
 143                 if (woff >= 0 && woff < ws_length)
 144                     ws[woff] = str;
 145             }
 146         }
 147     }
 148 }
 149
 150 #define FILE_MATCH_BLANK "\t "
 151
 152 static char *get_match_from_spec(ZebraHandle zh,
 153                           zebra_rec_keys_t reckeys,
 154                           const char *fname, const char *spec)
 155 {
 156     static char dstBuf[2048];      /* static here ??? */
 157     char *dst = dstBuf;
 158     const char *s = spec;
 159
 160     while (1)
 161     {
 162         for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
 163             ;
 164         if (!*s)
 165             break;
 166         if (*s == '(')
 167         {
 168             const char *ws[32];
 169             char attset_str[64], attname_str[64];
 170             int i;
 171             int first = 1;
 172
 173             for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 174                 ;
 175             for (i = 0; *s && *s != ',' && *s != ')' &&
 176                      !strchr(FILE_MATCH_BLANK, *s); s++)
 177                 if (i+1 < sizeof(attset_str))
 178                     attset_str[i++] = *s;
 179             attset_str[i] = '\0';
 180
 181             for (; strchr(FILE_MATCH_BLANK, *s); s++)
 182                 ;
 183             if (*s != ',')
 184                 strcpy(attname_str, attset_str);
 185             else
 186             {
 187                 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 188                     ;
 189                 for (i = 0; *s && *s != ')' &&
 190                          !strchr(FILE_MATCH_BLANK, *s); s++)
 191                     if (i+1 < sizeof(attname_str))
 192                         attname_str[i++] = *s;
 193                 attname_str[i] = '\0';
 194             }
 195
 196             searchRecordKey (zh, reckeys, attname_str, ws, 32);
 197
 198             if (*s != ')')
 199             {
 200                 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
 201                       spec, zh->m_group ? zh->m_group : "none");
 202                 return NULL;
 203             }
 204             s++;
 205
 206             for (i = 0; i<32; i++)
 207                 if (ws[i])
 208                 {
 209                     if (first)
 210                     {
 211                         *dst++ = ' ';
 212                         first = 0;
 213                     }
 214                     strcpy (dst, ws[i]);
 215                     dst += strlen(ws[i]);
 216                 }
 217             if (first)
 218             {
 219                 yaz_log (YLOG_WARN, "Record didn't contain match"
 220                       " fields in (%s,%s)", attset_str, attname_str);
 221                 return NULL;
 222             }
 223         }
 224         else if (*s == '$')
 225         {
 226             int spec_len;
 227             char special[64];
 228             const char *spec_src = NULL;
 229             const char *s1 = ++s;
 230             while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
 231                 s1++;
 232
 233             spec_len = s1 - s;
 234             if (spec_len > sizeof(special)-1)
 235                 spec_len = sizeof(special)-1;
 236             memcpy (special, s, spec_len);
 237             special[spec_len] = '\0';
 238             s = s1;
 239
 240             if (!strcmp (special, "group"))
 241                 spec_src = zh->m_group;
 242             else if (!strcmp (special, "database"))
 243                 spec_src = zh->basenames[0];
 244             else if (!strcmp (special, "filename")) {
 245                 spec_src = fname;
 246             }
 247             else if (!strcmp (special, "type"))
 248                 spec_src = zh->m_record_type;
 249             else
 250                 spec_src = NULL;
 251             if (spec_src)
 252             {
 253                 strcpy (dst, spec_src);
 254                 dst += strlen (spec_src);
 255             }
 256         }
 257         else if (*s == '\"' || *s == '\'')
 258         {
 259             int stopMarker = *s++;
 260             char tmpString[64];
 261             int i = 0;
 262
 263             while (*s && *s != stopMarker)
 264             {
 265                 if (i+1 < sizeof(tmpString))
 266                     tmpString[i++] = *s++;
 267             }
 268             if (*s)
 269                 s++;
 270             tmpString[i] = '\0';
 271             strcpy (dst, tmpString);
 272             dst += strlen (tmpString);
 273         }
 274         else
 275         {
 276             yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
 277                   spec, zh->m_group ? zh->m_group : "none");
 278             return NULL;
 279         }
 280         *dst++ = 1;
 281     }
 282     if (dst == dstBuf)
 283     {
 284         yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
 285               fname, zh->m_group ? zh->m_group : "none");
 286         return NULL;
 287     }
 288     *dst = '\0';
 289     return dstBuf;
 290 }
 291
 292 struct recordLogInfo {
 293     const char *fname;
 294     int recordOffset;
 295     struct recordGroup *rGroup;
 296 };
 297
 298 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
 299 {
 300     int i;
 301     for (i = 0; i<256; i++)
 302     {
 303         if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
 304             ctrl->seqno[i] = 1;
 305         else
 306             ctrl->seqno[i] = 0;
 307     }
 308     ctrl->flagShowRecords = !zh->m_flag_rw;
 309 }
 310
 311 static void all_matches_add(struct recExtractCtrl *ctrl)
 312 {
 313     RecWord word;
 314     extract_init(ctrl, &word);
 315     word.index_name = "_ALLRECORDS";
 316     word.index_type = 'w';
 317     word.seqno = 1;
 318     extract_add_index_string (&word, zinfo_index_category_alwaysmatches,
 319                               "", 0);
 320 }
 321
 322 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
 323                              int deleteFlag)
 324 {
 325     ZEBRA_RES r = ZEBRA_OK;
 326     int i, fd;
 327     char gprefix[128];
 328     char ext[128];
 329     char ext_res[128];
 330     struct file_read_info *fi = 0;
 331     const char *original_record_type = 0;
 332     RecType recType;
 333     void *recTypeClientData;
 334     struct ZebraRecStream stream, *streamp;
 335
 336     zebra_init_log_level();
 337
 338     if (!zh->m_group || !*zh->m_group)
 339         *gprefix = '\0';
 340     else
 341         sprintf (gprefix, "%s.", zh->m_group);
 342
 343     yaz_log(log_level_extract, "zebra_extract_file %s", fname);
 344
 345     /* determine file extension */
 346     *ext = '\0';
 347     for (i = strlen(fname); --i >= 0; )
 348         if (fname[i] == '/')
 349             break;
 350         else if (fname[i] == '.')
 351         {
 352             strcpy (ext, fname+i+1);
 353             break;
 354         }
 355     /* determine file type - depending on extension */
 356     original_record_type = zh->m_record_type;
 357     if (!zh->m_record_type)
 358     {
 359         sprintf (ext_res, "%srecordType.%s", gprefix, ext);
 360         zh->m_record_type = res_get (zh->res, ext_res);
 361     }
 362     if (!zh->m_record_type)
 363     {
 364         check_log_limit(zh);
 365         if (zh->records_processed + zh->records_skipped
 366             < zh->m_file_verbose_limit)
 367             yaz_log (YLOG_LOG, "? %s", fname);
 368         zh->records_skipped++;
 369         return 0;
 370     }
 371     /* determine match criteria */
 372     if (!zh->m_record_id)
 373     {
 374         sprintf (ext_res, "%srecordId.%s", gprefix, ext);
 375         zh->m_record_id = res_get (zh->res, ext_res);
 376     }
 377
 378     if (!(recType =
 379           recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
 380                           &recTypeClientData)))
 381     {
 382         yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
 383         return ZEBRA_FAIL;
 384     }
 385
 386     switch(recType->version)
 387     {
 388     case 0:
 389         break;
 390     default:
 391         yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
 392     }
 393     if (sysno && deleteFlag)
 394     {
 395         streamp = 0;
 396         fi = 0;
 397     }
 398     else
 399     {
 400         char full_rep[1024];
 401
 402         if (zh->path_reg && !yaz_is_abspath (fname))
 403         {
 404             strcpy (full_rep, zh->path_reg);
 405             strcat (full_rep, "/");
 406             strcat (full_rep, fname);
 407         }
 408         else
 409             strcpy (full_rep, fname);
 410
 411         if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
 412         {
 413             yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
 414             zh->m_record_type = original_record_type;
 415             return ZEBRA_FAIL;
 416         }
 417         streamp = &stream;
 418         zebra_create_stream_fd(streamp, fd, 0);
 419     }
 420     while(1)
 421     {
 422         int more = 0;
 423         r = zebra_extract_record_stream(zh, streamp,
 424                                         deleteFlag,
 425                                         0, /* tst_mode */
 426                                         zh->m_record_type,
 427                                         sysno,
 428                                         0, /*match_criteria */
 429                                         fname,
 430                                         1, /* force_update */
 431                                         1, /* allow_update */
 432                                         recType, recTypeClientData, &more);
 433         if (!more)
 434             break;
 435         if (sysno)
 436         {
 437             break;
 438         }
 439     }
 440     if (streamp)
 441         stream.destroy(streamp);
 442     zh->m_record_type = original_record_type;
 443     return r;
 444 }
 445
 446 /*
 447   If sysno is provided, then it's used to identify the reocord.
 448   If not, and match_criteria is provided, then sysno is guessed
 449   If not, and a record is provided, then sysno is got from there
 450
 451  */
 452
 453 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
 454                                       const char *buf, size_t buf_size,
 455                                       int delete_flag,
 456                                       int test_mode,
 457                                       const char *recordType,
 458                                       zint *sysno,
 459                                       const char *match_criteria,
 460                                       const char *fname,
 461                                       int force_update,
 462                                       int allow_update)
 463 {
 464     struct ZebraRecStream stream;
 465     ZEBRA_RES res;
 466     void *clientData;
 467     RecType recType = 0;
 468     int more = 0;
 469
 470     if (recordType && *recordType)
 471     {
 472         yaz_log(log_level_extract,
 473                 "Record type explicitly specified: %s", recordType);
 474         recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
 475                                   &clientData);
 476     }
 477     else
 478     {
 479         if (!(zh->m_record_type))
 480         {
 481             yaz_log (YLOG_WARN, "No such record type defined");
 482             return ZEBRA_FAIL;
 483         }
 484         yaz_log(log_level_extract, "Get record type from rgroup: %s",
 485                 zh->m_record_type);
 486         recType = recType_byName (zh->reg->recTypes, zh->res,
 487                                   zh->m_record_type, &clientData);
 488         recordType = zh->m_record_type;
 489     }
 490
 491     if (!recType)
 492     {
 493         yaz_log (YLOG_WARN, "No such record type: %s", recordType);
 494         return ZEBRA_FAIL;
 495     }
 496
 497     zebra_create_stream_mem(&stream, buf, buf_size);
 498
 499     res = zebra_extract_record_stream(zh, &stream,
 500                                       delete_flag,
 501                                       test_mode,
 502                                       recordType,
 503                                       sysno,
 504                                       match_criteria,
 505                                       fname,
 506                                       force_update,
 507                                       allow_update,
 508                                       recType, clientData, &more);
 509     stream.destroy(&stream);
 510     return res;
 511 }
 512
 513
 514 ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
 515                                       struct ZebraRecStream *stream,
 516                                       int delete_flag,
 517                                       int test_mode,
 518                                       const char *recordType,
 519                                       zint *sysno,
 520                                       const char *match_criteria,
 521                                       const char *fname,
 522                                       int force_update,
 523                                       int allow_update,
 524                                       RecType recType,
 525                                       void *recTypeClientData,
 526                                       int *more)
 527
 528 {
 529     zint sysno0 = 0;
 530     RecordAttr *recordAttr;
 531     struct recExtractCtrl extractCtrl;
 532     int r;
 533     const char *matchStr = 0;
 534     Record rec;
 535     off_t start_offset = 0, end_offset = 0;
 536     const char *pr_fname = fname;  /* filename to print .. */
 537     int show_progress = zh->records_processed + zh->records_skipped
 538         < zh->m_file_verbose_limit ? 1:0;
 539
 540     zebra_init_log_level();
 541
 542     if (!pr_fname)
 543         pr_fname = "<no file>";  /* make it printable if file is omitted */
 544
 545     zebra_rec_keys_reset(zh->reg->keys);
 546     zebra_rec_keys_reset(zh->reg->sortKeys);
 547
 548     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
 549     {
 550         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
 551                                       zh->m_explain_database))
 552             return ZEBRA_FAIL;
 553     }
 554
 555     if (stream)
 556     {
 557         off_t null_offset = 0;
 558         extractCtrl.stream = stream;
 559
 560         start_offset = stream->tellf(stream);
 561
 562         extractCtrl.first_record = start_offset ? 0 : 1;
 563
 564         stream->endf(stream, &null_offset);;
 565
 566         extractCtrl.init = extract_init;
 567         extractCtrl.tokenAdd = extract_token_add;
 568         extractCtrl.schemaAdd = extract_schema_add;
 569         extractCtrl.dh = zh->reg->dh;
 570         extractCtrl.handle = zh;
 571         extractCtrl.match_criteria[0] = '\0';
 572         extractCtrl.staticrank = 0;
 573
 574         init_extractCtrl(zh, &extractCtrl);
 575
 576         extract_set_store_data_prepare(&extractCtrl);
 577
 578         r = (*recType->extract)(recTypeClientData, &extractCtrl);
 579
 580         switch (r)
 581         {
 582         case RECCTRL_EXTRACT_EOF:
 583             return ZEBRA_FAIL;
 584         case RECCTRL_EXTRACT_ERROR_GENERIC:
 585             /* error occured during extraction ... */
 586             yaz_log (YLOG_WARN, "extract error: generic");
 587             return ZEBRA_FAIL;
 588         case RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER:
 589             /* error occured during extraction ... */
 590             yaz_log (YLOG_WARN, "extract error: no such filter");
 591             return ZEBRA_FAIL;
 592         case RECCTRL_EXTRACT_SKIP:
 593             if (show_progress)
 594                 yaz_log (YLOG_LOG, "skip %s %s " ZINT_FORMAT,
 595                          recordType, pr_fname, (zint) start_offset);
 596             *more = 1;
 597
 598             end_offset = stream->endf(stream, 0);
 599             if (end_offset)
 600                 stream->seekf(stream, end_offset);
 601
 602             return ZEBRA_OK;
 603         case RECCTRL_EXTRACT_OK:
 604             break;
 605         default:
 606             yaz_log (YLOG_WARN, "extract error: unknown error: %d", r);
 607             return ZEBRA_FAIL;
 608         }
 609         end_offset = stream->endf(stream, 0);
 610         if (end_offset)
 611             stream->seekf(stream, end_offset);
 612         else
 613             end_offset = stream->tellf(stream);
 614
 615         all_matches_add(&extractCtrl);
 616
 617         if (extractCtrl.match_criteria[0])
 618             match_criteria = extractCtrl.match_criteria;
 619     }
 620
 621     *more = 1;
 622     if (!sysno)
 623     {
 624         sysno = &sysno0;
 625
 626         if (match_criteria && *match_criteria) {
 627             matchStr = match_criteria;
 628         } else {
 629             if (zh->m_record_id && *zh->m_record_id) {
 630                 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
 631                                                zh->m_record_id);
 632                 if (!matchStr)
 633                 {
 634                     yaz_log (YLOG_LOG, "error %s %s " ZINT_FORMAT, recordType,
 635                              pr_fname, (zint) start_offset);
 636                     return ZEBRA_FAIL;
 637                 }
 638             }
 639         }
 640         if (matchStr)
 641         {
 642             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 643             char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
 644                                           matchStr);
 645             if (rinfo)
 646             {
 647                 assert(*rinfo == sizeof(*sysno));
 648                 memcpy (sysno, rinfo+1, sizeof(*sysno));
 649             }
 650         }
 651     }
 652     if (zebra_rec_keys_empty(zh->reg->keys))
 653     {
 654         /* the extraction process returned no information - the record
 655            is probably empty - unless flagShowRecords is in use */
 656         if (test_mode)
 657             return ZEBRA_OK;
 658     }
 659
 660     if (! *sysno)
 661     {
 662         /* new record */
 663         if (delete_flag)
 664         {
 665             yaz_log (YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 666                          pr_fname, (zint) start_offset);
 667             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
 668             return ZEBRA_FAIL;
 669         }
 670         if (show_progress)
 671             yaz_log (YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
 672                      (zint) start_offset);
 673         rec = rec_new (zh->reg->records);
 674
 675         *sysno = rec->sysno;
 676
 677         recordAttr = rec_init_attr (zh->reg->zei, rec);
 678         if (extractCtrl.staticrank < 0)
 679         {
 680             yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0");
 681             extractCtrl.staticrank = 0;
 682         }
 683         recordAttr->staticrank = extractCtrl.staticrank;
 684
 685         if (matchStr)
 686         {
 687             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 688             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
 689                             sizeof(*sysno), sysno);
 690         }
 691
 692         extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 693         extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 694                                   recordAttr->staticrank);
 695         zh->records_inserted++;
 696     }
 697     else
 698     {
 699         /* record already exists */
 700         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 701         zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
 702         if (!allow_update)
 703         {
 704             yaz_log (YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
 705                          recordType, pr_fname, (zint) start_offset);
 706             logRecord(zh);
 707             return ZEBRA_FAIL;
 708         }
 709
 710         rec = rec_get (zh->reg->records, *sysno);
 711         assert (rec);
 712
 713         recordAttr = rec_init_attr (zh->reg->zei, rec);
 714
 715         /* decrease total size */
 716         zebraExplain_recordBytesIncrement (zh->reg->zei,
 717                                            - recordAttr->recordSize);
 718
 719         zebra_rec_keys_set_buf(delkeys,
 720                                rec->info[recInfo_delKeys],
 721                                rec->size[recInfo_delKeys],
 722                                0);
 723         zebra_rec_keys_set_buf(sortKeys,
 724                                rec->info[recInfo_sortKeys],
 725                                rec->size[recInfo_sortKeys],
 726                                0);
 727
 728         extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
 729         extract_flush_record_keys(zh, *sysno, 0, delkeys,
 730                                   recordAttr->staticrank);
 731         if (delete_flag)
 732         {
 733             /* record going to be deleted */
 734             if (zebra_rec_keys_empty(delkeys))
 735             {
 736                 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 737                         pr_fname, (zint) start_offset);
 738                 yaz_log(YLOG_WARN, "cannot delete file above, "
 739                         "storeKeys false (3)");
 740             }
 741             else
 742             {
 743                 if (show_progress)
 744                     yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 745                             pr_fname, (zint) start_offset);
 746                 zh->records_deleted++;
 747                 if (matchStr)
 748                 {
 749                     int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 750                     dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
 751                 }
 752                 rec_del (zh->reg->records, &rec);
 753             }
 754             rec_free(&rec);
 755             logRecord(zh);
 756             return ZEBRA_OK;
 757         }
 758         else
 759         {
 760             if (show_progress)
 761                 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 762                         pr_fname, (zint) start_offset);
 763             recordAttr->staticrank = extractCtrl.staticrank;
 764             extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 765             extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 766                                       recordAttr->staticrank);
 767             zh->records_updated++;
 768         }
 769         zebra_rec_keys_close(delkeys);
 770         zebra_rec_keys_close(sortKeys);
 771     }
 772     /* update file type */
 773     xfree (rec->info[recInfo_fileType]);
 774     rec->info[recInfo_fileType] =
 775         rec_strdup (recordType, &rec->size[recInfo_fileType]);
 776
 777     /* update filename */
 778     xfree (rec->info[recInfo_filename]);
 779     rec->info[recInfo_filename] =
 780         rec_strdup (fname, &rec->size[recInfo_filename]);
 781
 782     /* update delete keys */
 783     xfree (rec->info[recInfo_delKeys]);
 784     if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
 785     {
 786         zebra_rec_keys_get_buf(zh->reg->keys,
 787                                &rec->info[recInfo_delKeys],
 788                                &rec->size[recInfo_delKeys]);
 789     }
 790     else
 791     {
 792         rec->info[recInfo_delKeys] = NULL;
 793         rec->size[recInfo_delKeys] = 0;
 794     }
 795     /* update sort keys */
 796     xfree (rec->info[recInfo_sortKeys]);
 797
 798     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 799                            &rec->info[recInfo_sortKeys],
 800                            &rec->size[recInfo_sortKeys]);
 801
 802     if (stream)
 803     {
 804         recordAttr->recordSize = end_offset - start_offset;
 805         zebraExplain_recordBytesIncrement(zh->reg->zei,
 806                                           recordAttr->recordSize);
 807     }
 808
 809     /* set run-number for this record */
 810     recordAttr->runNumber =
 811         zebraExplain_runNumberIncrement (zh->reg->zei, 0);
 812
 813     /* update store data */
 814     xfree (rec->info[recInfo_storeData]);
 815
 816     /* update store data */
 817     if (zh->store_data_buf)
 818     {
 819         rec->size[recInfo_storeData] = zh->store_data_size;
 820         rec->info[recInfo_storeData] = zh->store_data_buf;
 821         zh->store_data_buf = 0;
 822         recordAttr->recordSize = zh->store_data_size;
 823     }
 824     else if (zh->m_store_data)
 825     {
 826         off_t cur_offset = stream->tellf(stream);
 827
 828         rec->size[recInfo_storeData] = recordAttr->recordSize;
 829         rec->info[recInfo_storeData] = (char *)
 830             xmalloc (recordAttr->recordSize);
 831         stream->seekf(stream, start_offset);
 832         stream->readf(stream, rec->info[recInfo_storeData],
 833                       recordAttr->recordSize);
 834         stream->seekf(stream, cur_offset);
 835     }
 836     else
 837     {
 838         rec->info[recInfo_storeData] = NULL;
 839         rec->size[recInfo_storeData] = 0;
 840     }
 841     /* update database name */
 842     xfree (rec->info[recInfo_databaseName]);
 843     rec->info[recInfo_databaseName] =
 844         rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
 845
 846     /* update offset */
 847     recordAttr->recordOffset = start_offset;
 848
 849     /* commit this record */
 850     rec_put (zh->reg->records, &rec);
 851     logRecord(zh);
 852     return ZEBRA_OK;
 853 }
 854
 855 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
 856 {
 857     ZebraHandle zh = (ZebraHandle) handle;
 858     struct recExtractCtrl extractCtrl;
 859
 860     if (zebraExplain_curDatabase (zh->reg->zei,
 861                                   rec->info[recInfo_databaseName]))
 862     {
 863         abort();
 864         if (zebraExplain_newDatabase (zh->reg->zei,
 865                                       rec->info[recInfo_databaseName], 0))
 866             abort ();
 867     }
 868
 869     zebra_rec_keys_reset(zh->reg->keys);
 870     zebra_rec_keys_reset(zh->reg->sortKeys);
 871
 872     extractCtrl.init = extract_init;
 873     extractCtrl.tokenAdd = extract_token_add;
 874     extractCtrl.schemaAdd = extract_schema_add;
 875     extractCtrl.dh = zh->reg->dh;
 876
 877     init_extractCtrl(zh, &extractCtrl);
 878
 879     extractCtrl.flagShowRecords = 0;
 880     extractCtrl.match_criteria[0] = '\0';
 881     extractCtrl.staticrank = 0;
 882     extractCtrl.handle = handle;
 883     extractCtrl.first_record = 1;
 884
 885     extract_set_store_data_prepare(&extractCtrl);
 886
 887     if (n)
 888         grs_extract_tree(&extractCtrl, n);
 889
 890     if (rec->size[recInfo_delKeys])
 891     {
 892         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 893
 894         zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
 895
 896         zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
 897                                rec->size[recInfo_delKeys],
 898                                0);
 899         extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
 900         zebra_rec_keys_close(delkeys);
 901
 902         zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
 903                                rec->size[recInfo_sortKeys],
 904                                0);
 905
 906         extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
 907         zebra_rec_keys_close(sortkeys);
 908     }
 909     extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
 910     extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
 911
 912     xfree (rec->info[recInfo_delKeys]);
 913     zebra_rec_keys_get_buf(zh->reg->keys,
 914                            &rec->info[recInfo_delKeys],
 915                            &rec->size[recInfo_delKeys]);
 916
 917     xfree (rec->info[recInfo_sortKeys]);
 918     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 919                            &rec->info[recInfo_sortKeys],
 920                            &rec->size[recInfo_sortKeys]);
 921     return ZEBRA_OK;
 922 }
 923
 924 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
 925                           zebra_rec_keys_t reckeys,
 926                           int level)
 927 {
 928     if (zebra_rec_keys_rewind(reckeys))
 929     {
 930         size_t slen;
 931         const char *str;
 932         struct it_key key;
 933         NMEM nmem = nmem_create();
 934
 935         while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
 936         {
 937             char keystr[200]; /* room for zints to print */
 938             char *dst_term = 0;
 939             int ord = CAST_ZINT_TO_INT(key.mem[0]);
 940             int index_type, i;
 941             const char *string_index;
 942
 943             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
 944                                     0/* db */, &string_index);
 945             assert(index_type);
 946             zebra_term_untrans_iconv(zh, nmem, index_type,
 947                                      &dst_term, str);
 948             *keystr = '\0';
 949             for (i = 0; i<key.len; i++)
 950             {
 951                 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key.mem[i]);
 952             }
 953
 954             if (*str < CHR_BASE_CHAR)
 955             {
 956                 int i;
 957                 char dst_buf[200]; /* room for special chars */
 958
 959                 strcpy(dst_buf , "?");
 960
 961                 if (!strcmp(str, ""))
 962                     strcpy(dst_buf, "alwaysmatches");
 963                 if (!strcmp(str, FIRST_IN_FIELD_STR))
 964                     strcpy(dst_buf, "firstinfield");
 965                 else if (!strcmp(str, CHR_UNKNOWN))
 966                     strcpy(dst_buf, "unknown");
 967                 else if (!strcmp(str, CHR_SPACE))
 968                     strcpy(dst_buf, "space");
 969
 970                 for (i = 0; i<slen; i++)
 971                 {
 972                     sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
 973                 }
 974                 yaz_log(level, "%s%c %s %s", keystr, index_type,
 975                         string_index, dst_buf);
 976
 977             }
 978             else
 979                 yaz_log(level, "%s%c %s \"%s\"", keystr, index_type,
 980                         string_index, dst_term);
 981
 982             nmem_reset(nmem);
 983         }
 984         nmem_destroy(nmem);
 985     }
 986 }
 987
 988 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
 989                              zebra_rec_keys_t reckeys)
 990 {
 991     ZebraExplainInfo zei = zh->reg->zei;
 992     struct ord_stat {
 993         int no;
 994         int ord;
 995         struct ord_stat *next;
 996     };
 997
 998     if (zebra_rec_keys_rewind(reckeys))
 999     {
1000         struct ord_stat *ord_list = 0;
1001         struct ord_stat *p;
1002         size_t slen;
1003         const char *str;
1004         struct it_key key_in;
1005         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1006         {
1007             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1008
1009             for (p = ord_list; p ; p = p->next)
1010                 if (p->ord == ord)
1011                 {
1012                     p->no++;
1013                     break;
1014                 }
1015             if (!p)
1016             {
1017                 p = xmalloc(sizeof(*p));
1018                 p->no = 1;
1019                 p->ord = ord;
1020                 p->next = ord_list;
1021                 ord_list = p;
1022             }
1023         }
1024
1025         p = ord_list;
1026         while (p)
1027         {
1028             struct ord_stat *p1 = p;
1029
1030             if (is_insert)
1031                 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
1032             else
1033                 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1034             p = p->next;
1035             xfree(p1);
1036         }
1037     }
1038 }
1039
1040 void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd,
1041                                zebra_rec_keys_t reckeys,
1042                                zint staticrank)
1043 {
1044     ZebraExplainInfo zei = zh->reg->zei;
1045
1046     extract_rec_keys_adjust(zh, cmd, reckeys);
1047
1048     if (log_level_details)
1049     {
1050         yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1051                 sysno, cmd ? "insert" : "delete");
1052         extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1053     }
1054
1055     if (!zh->reg->key_block)
1056     {
1057         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1058         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1059         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1060         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1061     }
1062     zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1063
1064     if (zebra_rec_keys_rewind(reckeys))
1065     {
1066         size_t slen;
1067         const char *str;
1068         struct it_key key_in;
1069         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1070         {
1071             key_block_write(zh->reg->key_block, sysno,
1072                             &key_in, cmd, str, slen,
1073                             staticrank, zh->m_staticrank);
1074         }
1075     }
1076 }
1077
1078
1079 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
1080                                      zebra_rec_keys_t reckeys,
1081                                      zebra_snippets *snippets)
1082 {
1083     NMEM nmem = nmem_create();
1084     if (zebra_rec_keys_rewind(reckeys))
1085     {
1086         const char *str;
1087         size_t slen;
1088         struct it_key key;
1089         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1090         {
1091             char *dst_term = 0;
1092             int ord;
1093             zint seqno;
1094             int index_type;
1095
1096             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1097             seqno = key.mem[key.len-1];
1098             ord = CAST_ZINT_TO_INT(key.mem[0]);
1099
1100             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1101                                     0/* db */, 0 /* string_index */);
1102             assert(index_type);
1103             zebra_term_untrans_iconv(zh, nmem, index_type,
1104                                      &dst_term, str);
1105             zebra_snippets_append(snippets, seqno, ord, dst_term);
1106             nmem_reset(nmem);
1107         }
1108     }
1109     nmem_destroy(nmem);
1110     return ZEBRA_OK;
1111 }
1112
1113 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1114 {
1115     yaz_log(YLOG_LOG, "print_rec_keys");
1116     if (zebra_rec_keys_rewind(reckeys))
1117     {
1118         const char *str;
1119         size_t slen;
1120         struct it_key key;
1121         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1122         {
1123             char dst_buf[IT_MAX_WORD];
1124             zint seqno;
1125             int index_type;
1126             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1127             const char *db = 0;
1128             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1129
1130             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1131
1132             seqno = key.mem[key.len-1];
1133
1134             zebra_term_untrans(zh, index_type, dst_buf, str);
1135
1136             yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1137                     " term=%s", ord, seqno, dst_buf);
1138         }
1139     }
1140 }
1141
1142 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1143                                      const char *str, int length)
1144 {
1145     struct it_key key;
1146     ZebraHandle zh = p->extractCtrl->handle;
1147     ZebraExplainInfo zei = zh->reg->zei;
1148     int ch, i;
1149
1150     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1151     if (ch < 0)
1152         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1153
1154     i = 0;
1155     key.mem[i++] = ch;
1156     key.mem[i++] = p->record_id;
1157     key.mem[i++] = p->section_id;
1158
1159     if (zh->m_segment_indexing)
1160         key.mem[i++] = p->segment;
1161     key.mem[i++] = p->seqno;
1162     key.len = i;
1163
1164     zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1165 }
1166
1167 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1168 {
1169     struct it_key key;
1170     ZebraHandle zh = p->extractCtrl->handle;
1171     ZebraExplainInfo zei = zh->reg->zei;
1172     int ch;
1173     zinfo_index_category_t cat = zinfo_index_category_sort;
1174
1175     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1176     if (ch < 0)
1177         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1178     key.len = 2;
1179     key.mem[0] = ch;
1180     key.mem[1] = p->record_id;
1181
1182     zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1183 }
1184
1185 static void extract_add_staticrank_string(RecWord *p,
1186                                           const char *str, int length)
1187 {
1188     char valz[40];
1189     struct recExtractCtrl *ctrl = p->extractCtrl;
1190
1191     if (length > sizeof(valz)-1)
1192         length = sizeof(valz)-1;
1193
1194     memcpy(valz, str, length);
1195     valz[length] = '\0';
1196     ctrl->staticrank = atozint(valz);
1197 }
1198
1199 static void extract_add_string(RecWord *p, const char *string, int length)
1200 {
1201     ZebraHandle zh = p->extractCtrl->handle;
1202     assert (length > 0);
1203
1204     if (!p->index_name)
1205         return;
1206
1207     if (zebra_maps_is_index(zh->reg->zebra_maps, p->index_type))
1208     {
1209         extract_add_index_string(p, zinfo_index_category_index,
1210                                  string, length);
1211         if (zebra_maps_is_alwaysmatches(zh->reg->zebra_maps, p->index_type))
1212         {
1213             RecWord word;
1214             memcpy(&word, p, sizeof(word));
1215
1216             word.seqno = 1;
1217             extract_add_index_string(
1218                 &word, zinfo_index_category_alwaysmatches, "", 0);
1219         }
1220     }
1221     else if (zebra_maps_is_sort(zh->reg->zebra_maps, p->index_type))
1222     {
1223         extract_add_sort_string(p, string, length);
1224     }
1225     else if (zebra_maps_is_staticrank(zh->reg->zebra_maps, p->index_type))
1226     {
1227         extract_add_staticrank_string(p, string, length);
1228     }
1229 }
1230
1231 static void extract_add_incomplete_field(RecWord *p)
1232 {
1233     ZebraHandle zh = p->extractCtrl->handle;
1234     const char *b = p->term_buf;
1235     int remain = p->term_len;
1236     int first = 1;
1237     const char **map = 0;
1238
1239     if (remain > 0)
1240         map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1241
1242     while (map)
1243     {
1244         char buf[IT_MAX_WORD+1];
1245         int i, remain;
1246
1247         /* Skip spaces */
1248         while (map && *map && **map == *CHR_SPACE)
1249         {
1250             remain = p->term_len - (b - p->term_buf);
1251             if (remain > 0)
1252                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b,
1253                                        remain, 0);
1254             else
1255                 map = 0;
1256         }
1257         if (!map)
1258             break;
1259         i = 0;
1260         while (map && *map && **map != *CHR_SPACE)
1261         {
1262             const char *cp = *map;
1263
1264             while (i < IT_MAX_WORD && *cp)
1265                 buf[i++] = *(cp++);
1266             remain = p->term_len - (b - p->term_buf);
1267             if (remain > 0)
1268                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1269             else
1270                 map = 0;
1271         }
1272         if (!i)
1273             return;
1274
1275         if (first)
1276         {
1277             first = 0;
1278             if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type))
1279             {
1280                 /* first in field marker */
1281                 extract_add_string(p, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1282                 p->seqno++;
1283             }
1284         }
1285         extract_add_string (p, buf, i);
1286         p->seqno++;
1287     }
1288 }
1289
1290 static void extract_add_complete_field (RecWord *p)
1291 {
1292     ZebraHandle zh = p->extractCtrl->handle;
1293     const char *b = p->term_buf;
1294     char buf[IT_MAX_WORD+1];
1295     const char **map = 0;
1296     int i = 0, remain = p->term_len;
1297
1298     if (remain > 0)
1299         map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, remain, 1);
1300
1301     while (remain > 0 && i < IT_MAX_WORD)
1302     {
1303         while (map && *map && **map == *CHR_SPACE)
1304         {
1305             remain = p->term_len - (b - p->term_buf);
1306
1307             if (remain > 0)
1308             {
1309                 int first = i ? 0 : 1;  /* first position */
1310                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, first);
1311             }
1312             else
1313                 map = 0;
1314         }
1315         if (!map)
1316             break;
1317
1318         if (i && i < IT_MAX_WORD)
1319             buf[i++] = *CHR_SPACE;
1320         while (map && *map && **map != *CHR_SPACE)
1321         {
1322             const char *cp = *map;
1323
1324             if (**map == *CHR_CUT)
1325             {
1326                 i = 0;
1327             }
1328             else
1329             {
1330                 if (i >= IT_MAX_WORD)
1331                     break;
1332                 while (i < IT_MAX_WORD && *cp)
1333                     buf[i++] = *(cp++);
1334             }
1335             remain = p->term_len  - (b - p->term_buf);
1336             if (remain > 0)
1337             {
1338                 map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b,
1339                                         remain, 0);
1340             }
1341             else
1342                 map = 0;
1343         }
1344     }
1345     if (!i)
1346         return;
1347     extract_add_string (p, buf, i);
1348 }
1349
1350 static void extract_token_add(RecWord *p)
1351 {
1352     ZebraHandle zh = p->extractCtrl->handle;
1353     WRBUF wrbuf;
1354
1355     if (log_level_extract)
1356     {
1357         yaz_log(log_level_extract, "extract_token_add "
1358                 "type=%c index=%s seqno=" ZINT_FORMAT " s=%.*s",
1359                 p->index_type, p->index_name,
1360                 p->seqno, p->term_len, p->term_buf);
1361     }
1362     if ((wrbuf = zebra_replace(zh->reg->zebra_maps, p->index_type, 0,
1363                                p->term_buf, p->term_len)))
1364     {
1365         p->term_buf = wrbuf_buf(wrbuf);
1366         p->term_len = wrbuf_len(wrbuf);
1367     }
1368     if (zebra_maps_is_complete (zh->reg->zebra_maps, p->index_type))
1369         extract_add_complete_field (p);
1370     else
1371         extract_add_incomplete_field(p);
1372 }
1373
1374 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1375                                       void *buf, size_t sz)
1376 {
1377     ZebraHandle zh = (ZebraHandle) p->handle;
1378
1379     xfree(zh->store_data_buf);
1380     zh->store_data_buf = 0;
1381     zh->store_data_size = 0;
1382     if (buf && sz)
1383     {
1384         zh->store_data_buf = xmalloc(sz);
1385         zh->store_data_size = sz;
1386         memcpy(zh->store_data_buf, buf, sz);
1387     }
1388 }
1389
1390 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1391 {
1392     ZebraHandle zh = (ZebraHandle) p->handle;
1393     xfree(zh->store_data_buf);
1394     zh->store_data_buf = 0;
1395     zh->store_data_size = 0;
1396     p->setStoreData = extract_set_store_data_cb;
1397 }
1398
1399 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1400 {
1401     ZebraHandle zh = (ZebraHandle) p->handle;
1402     zebraExplain_addSchema (zh->reg->zei, oid);
1403 }
1404
1405 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1406                              int cmd, zebra_rec_keys_t reckeys)
1407 {
1408 #if 0
1409     yaz_log(YLOG_LOG, "extract_flush_sort_keys cmd=%d sysno=" ZINT_FORMAT,
1410             cmd, sysno);
1411     extract_rec_keys_log(zh, cmd, reckeys, YLOG_LOG);
1412 #endif
1413
1414     if (zebra_rec_keys_rewind(reckeys))
1415     {
1416         zebra_sort_index_t si = zh->reg->sort_index;
1417         size_t slen;
1418         const char *str;
1419         struct it_key key_in;
1420
1421         zebra_sort_sysno(si, sysno);
1422
1423         while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1424         {
1425             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1426
1427             zebra_sort_type(si, ord);
1428             if (cmd == 1)
1429                 zebra_sort_add(si, str, slen);
1430             else
1431                 zebra_sort_delete(si);
1432         }
1433     }
1434 }
1435
1436 /*
1437  * Local variables:
1438  * c-basic-offset: 4
1439  * indent-tabs-mode: nil
1440  * End:
1441  * vim: shiftwidth=4 tabstop=8 expandtab
1442  */
1443