index/extract.c

   1 /* $Id: extract.c,v 1.255 2007-04-07 22:26:27 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #endif
  29 #if HAVE_UNISTD_H
  30 #include <unistd.h>
  31 #endif
  32 #include <fcntl.h>
  33
  34 #include "index.h"
  35 #include "orddict.h"
  36 #include <direntz.h>
  37 #include <charmap.h>
  38
  39 static int log_level_extract = 0;
  40 static int log_level_details = 0;
  41 static int log_level_initialized = 0;
  42
  43 /* 1 if we use eliminitate identical delete/insert keys */
  44 /* eventually this the 0-case code will be removed */
  45 #define FLUSH2 1
  46
  47 void extract_flush_record_keys2(ZebraHandle zh, zint sysno,
  48                                 zebra_rec_keys_t ins_keys,
  49                                 zint ins_rank,
  50                                 zebra_rec_keys_t del_keys,
  51                                 zint del_rank);
  52
  53 static void zebra_init_log_level(void)
  54 {
  55     if (!log_level_initialized)
  56     {
  57         log_level_initialized = 1;
  58
  59         log_level_extract = yaz_log_module_level("extract");
  60         log_level_details = yaz_log_module_level("indexdetails");
  61     }
  62 }
  63
  64 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
  65                                       int cmd, zebra_rec_keys_t reckeys,
  66                                       zint staticrank);
  67 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
  68                                     int cmd, zebra_rec_keys_t skp);
  69 static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid);
  70 static void extract_token_add (RecWord *p);
  71
  72 static void check_log_limit(ZebraHandle zh)
  73 {
  74     if (zh->records_processed + zh->records_skipped == zh->m_file_verbose_limit)
  75     {
  76         yaz_log(YLOG_LOG, "More than %d file log entries. Omitting rest",
  77                 zh->m_file_verbose_limit);
  78     }
  79 }
  80
  81 static void logRecord (ZebraHandle zh)
  82 {
  83     check_log_limit(zh);
  84     ++zh->records_processed;
  85     if (!(zh->records_processed % 1000))
  86     {
  87         yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
  88                 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
  89                 zh->records_processed, zh->records_inserted,
  90                 zh->records_updated, zh->records_deleted);
  91     }
  92 }
  93
  94 static void extract_add_index_string (RecWord *p,
  95                                       zinfo_index_category_t cat,
  96                                       const char *str, int length);
  97
  98 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
  99
 100 static void extract_init(struct recExtractCtrl *p, RecWord *w)
 101 {
 102     w->seqno = 1;
 103     w->index_name = "any";
 104     w->index_type = 'w';
 105     w->extractCtrl = p;
 106     w->record_id = 0;
 107     w->section_id = 0;
 108     w->segment = 0;
 109 }
 110
 111 static void searchRecordKey(ZebraHandle zh,
 112                             zebra_rec_keys_t reckeys,
 113                             const char *index_name,
 114                             const char **ws, int ws_length)
 115 {
 116     int i;
 117     int ch = -1;
 118     zinfo_index_category_t cat = zinfo_index_category_index;
 119
 120     for (i = 0; i<ws_length; i++)
 121         ws[i] = NULL;
 122
 123     if (ch < 0)
 124         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, '0', index_name);
 125     if (ch < 0)
 126         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'p', index_name);
 127     if (ch < 0)
 128         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'w', index_name);
 129
 130     if (ch < 0)
 131         return ;
 132
 133     if (zebra_rec_keys_rewind(reckeys))
 134     {
 135         zint startSeq = -1;
 136         const char *str;
 137         size_t slen;
 138         struct it_key key;
 139         zint seqno;
 140         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
 141         {
 142             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
 143
 144             seqno = key.mem[key.len-1];
 145
 146             if (key.mem[0] == ch)
 147             {
 148                 zint woff;
 149
 150                 if (startSeq == -1)
 151                     startSeq = seqno;
 152                 woff = seqno - startSeq;
 153                 if (woff >= 0 && woff < ws_length)
 154                     ws[woff] = str;
 155             }
 156         }
 157     }
 158 }
 159
 160 #define FILE_MATCH_BLANK "\t "
 161
 162 static char *get_match_from_spec(ZebraHandle zh,
 163                           zebra_rec_keys_t reckeys,
 164                           const char *fname, const char *spec)
 165 {
 166     static char dstBuf[2048];      /* static here ??? */
 167     char *dst = dstBuf;
 168     const char *s = spec;
 169
 170     while (1)
 171     {
 172         for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
 173             ;
 174         if (!*s)
 175             break;
 176         if (*s == '(')
 177         {
 178             const char *ws[32];
 179             char attset_str[64], attname_str[64];
 180             int i;
 181             int first = 1;
 182
 183             for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 184                 ;
 185             for (i = 0; *s && *s != ',' && *s != ')' &&
 186                      !strchr(FILE_MATCH_BLANK, *s); s++)
 187                 if (i+1 < sizeof(attset_str))
 188                     attset_str[i++] = *s;
 189             attset_str[i] = '\0';
 190
 191             for (; strchr(FILE_MATCH_BLANK, *s); s++)
 192                 ;
 193             if (*s != ',')
 194                 strcpy(attname_str, attset_str);
 195             else
 196             {
 197                 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 198                     ;
 199                 for (i = 0; *s && *s != ')' &&
 200                          !strchr(FILE_MATCH_BLANK, *s); s++)
 201                     if (i+1 < sizeof(attname_str))
 202                         attname_str[i++] = *s;
 203                 attname_str[i] = '\0';
 204             }
 205
 206             searchRecordKey (zh, reckeys, attname_str, ws, 32);
 207
 208             if (*s != ')')
 209             {
 210                 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
 211                       spec, zh->m_group ? zh->m_group : "none");
 212                 return NULL;
 213             }
 214             s++;
 215
 216             for (i = 0; i<32; i++)
 217                 if (ws[i])
 218                 {
 219                     if (first)
 220                     {
 221                         *dst++ = ' ';
 222                         first = 0;
 223                     }
 224                     strcpy (dst, ws[i]);
 225                     dst += strlen(ws[i]);
 226                 }
 227             if (first)
 228             {
 229                 yaz_log (YLOG_WARN, "Record didn't contain match"
 230                       " fields in (%s,%s)", attset_str, attname_str);
 231                 return NULL;
 232             }
 233         }
 234         else if (*s == '$')
 235         {
 236             int spec_len;
 237             char special[64];
 238             const char *spec_src = NULL;
 239             const char *s1 = ++s;
 240             while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
 241                 s1++;
 242
 243             spec_len = s1 - s;
 244             if (spec_len > sizeof(special)-1)
 245                 spec_len = sizeof(special)-1;
 246             memcpy (special, s, spec_len);
 247             special[spec_len] = '\0';
 248             s = s1;
 249
 250             if (!strcmp (special, "group"))
 251                 spec_src = zh->m_group;
 252             else if (!strcmp (special, "database"))
 253                 spec_src = zh->basenames[0];
 254             else if (!strcmp (special, "filename")) {
 255                 spec_src = fname;
 256             }
 257             else if (!strcmp (special, "type"))
 258                 spec_src = zh->m_record_type;
 259             else
 260                 spec_src = NULL;
 261             if (spec_src)
 262             {
 263                 strcpy (dst, spec_src);
 264                 dst += strlen (spec_src);
 265             }
 266         }
 267         else if (*s == '\"' || *s == '\'')
 268         {
 269             int stopMarker = *s++;
 270             char tmpString[64];
 271             int i = 0;
 272
 273             while (*s && *s != stopMarker)
 274             {
 275                 if (i+1 < sizeof(tmpString))
 276                     tmpString[i++] = *s++;
 277             }
 278             if (*s)
 279                 s++;
 280             tmpString[i] = '\0';
 281             strcpy (dst, tmpString);
 282             dst += strlen (tmpString);
 283         }
 284         else
 285         {
 286             yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
 287                   spec, zh->m_group ? zh->m_group : "none");
 288             return NULL;
 289         }
 290         *dst++ = 1;
 291     }
 292     if (dst == dstBuf)
 293     {
 294         yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
 295               fname, zh->m_group ? zh->m_group : "none");
 296         return NULL;
 297     }
 298     *dst = '\0';
 299     return dstBuf;
 300 }
 301
 302 struct recordLogInfo {
 303     const char *fname;
 304     int recordOffset;
 305     struct recordGroup *rGroup;
 306 };
 307
 308 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
 309 {
 310     int i;
 311     for (i = 0; i<256; i++)
 312     {
 313         if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
 314             ctrl->seqno[i] = 1;
 315         else
 316             ctrl->seqno[i] = 0;
 317     }
 318     ctrl->flagShowRecords = !zh->m_flag_rw;
 319 }
 320
 321 static void all_matches_add(struct recExtractCtrl *ctrl)
 322 {
 323     RecWord word;
 324     extract_init(ctrl, &word);
 325     word.index_name = "_ALLRECORDS";
 326     word.index_type = 'w';
 327     word.seqno = 1;
 328     extract_add_index_string (&word, zinfo_index_category_alwaysmatches,
 329                               "", 0);
 330 }
 331
 332 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
 333                                        struct ZebraRecStream *stream,
 334                                        enum zebra_recctrl_action_t action,
 335                                        int test_mode,
 336                                        const char *recordType,
 337                                        zint *sysno,
 338                                        const char *match_criteria,
 339                                        const char *fname,
 340                                        RecType recType,
 341                                        void *recTypeClientData);
 342
 343
 344 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
 345                              int deleteFlag)
 346 {
 347     ZEBRA_RES r = ZEBRA_OK;
 348     int i, fd;
 349     char gprefix[128];
 350     char ext[128];
 351     char ext_res[128];
 352     struct file_read_info *fi = 0;
 353     const char *original_record_type = 0;
 354     RecType recType;
 355     void *recTypeClientData;
 356     struct ZebraRecStream stream, *streamp;
 357
 358     zebra_init_log_level();
 359
 360     if (!zh->m_group || !*zh->m_group)
 361         *gprefix = '\0';
 362     else
 363         sprintf (gprefix, "%s.", zh->m_group);
 364
 365     yaz_log(log_level_extract, "zebra_extract_file %s", fname);
 366
 367     /* determine file extension */
 368     *ext = '\0';
 369     for (i = strlen(fname); --i >= 0; )
 370         if (fname[i] == '/')
 371             break;
 372         else if (fname[i] == '.')
 373         {
 374             strcpy (ext, fname+i+1);
 375             break;
 376         }
 377     /* determine file type - depending on extension */
 378     original_record_type = zh->m_record_type;
 379     if (!zh->m_record_type)
 380     {
 381         sprintf (ext_res, "%srecordType.%s", gprefix, ext);
 382         zh->m_record_type = res_get (zh->res, ext_res);
 383     }
 384     if (!zh->m_record_type)
 385     {
 386         check_log_limit(zh);
 387         if (zh->records_processed + zh->records_skipped
 388             < zh->m_file_verbose_limit)
 389             yaz_log (YLOG_LOG, "? %s", fname);
 390         zh->records_skipped++;
 391         return 0;
 392     }
 393     /* determine match criteria */
 394     if (!zh->m_record_id)
 395     {
 396         sprintf (ext_res, "%srecordId.%s", gprefix, ext);
 397         zh->m_record_id = res_get (zh->res, ext_res);
 398     }
 399
 400     if (!(recType =
 401           recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
 402                           &recTypeClientData)))
 403     {
 404         yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
 405         return ZEBRA_FAIL;
 406     }
 407
 408     switch(recType->version)
 409     {
 410     case 0:
 411         break;
 412     default:
 413         yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
 414     }
 415     if (sysno && deleteFlag)
 416     {
 417         streamp = 0;
 418         fi = 0;
 419     }
 420     else
 421     {
 422         char full_rep[1024];
 423
 424         if (zh->path_reg && !yaz_is_abspath (fname))
 425         {
 426             strcpy (full_rep, zh->path_reg);
 427             strcat (full_rep, "/");
 428             strcat (full_rep, fname);
 429         }
 430         else
 431             strcpy (full_rep, fname);
 432
 433         if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
 434         {
 435             yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
 436             zh->m_record_type = original_record_type;
 437             return ZEBRA_FAIL;
 438         }
 439         streamp = &stream;
 440         zebra_create_stream_fd(streamp, fd, 0);
 441     }
 442     r = zebra_extract_records_stream(zh, streamp,
 443                                      deleteFlag ?
 444                                      action_delete : action_update,
 445                                      0, /* tst_mode */
 446                                      zh->m_record_type,
 447                                      sysno,
 448                                      0, /*match_criteria */
 449                                      fname,
 450                                      recType, recTypeClientData);
 451     if (streamp)
 452         stream.destroy(streamp);
 453     zh->m_record_type = original_record_type;
 454     return r;
 455 }
 456
 457 /*
 458   If sysno is provided, then it's used to identify the reocord.
 459   If not, and match_criteria is provided, then sysno is guessed
 460   If not, and a record is provided, then sysno is got from there
 461
 462  */
 463
 464 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
 465                                       const char *buf, size_t buf_size,
 466                                       enum zebra_recctrl_action_t action,
 467                                       int test_mode,
 468                                       const char *recordType,
 469                                       zint *sysno,
 470                                       const char *match_criteria,
 471                                       const char *fname)
 472 {
 473     struct ZebraRecStream stream;
 474     ZEBRA_RES res;
 475     void *clientData;
 476     RecType recType = 0;
 477
 478     if (recordType && *recordType)
 479     {
 480         yaz_log(log_level_extract,
 481                 "Record type explicitly specified: %s", recordType);
 482         recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
 483                                   &clientData);
 484     }
 485     else
 486     {
 487         if (!(zh->m_record_type))
 488         {
 489             yaz_log (YLOG_WARN, "No such record type defined");
 490             return ZEBRA_FAIL;
 491         }
 492         yaz_log(log_level_extract, "Get record type from rgroup: %s",
 493                 zh->m_record_type);
 494         recType = recType_byName (zh->reg->recTypes, zh->res,
 495                                   zh->m_record_type, &clientData);
 496         recordType = zh->m_record_type;
 497     }
 498
 499     if (!recType)
 500     {
 501         yaz_log (YLOG_WARN, "No such record type: %s", recordType);
 502         return ZEBRA_FAIL;
 503     }
 504
 505     zebra_create_stream_mem(&stream, buf, buf_size);
 506
 507     res = zebra_extract_records_stream(zh, &stream,
 508                                        action,
 509                                        test_mode,
 510                                        recordType,
 511                                        sysno,
 512                                        match_criteria,
 513                                        fname,
 514                                        recType, clientData);
 515     stream.destroy(&stream);
 516     return res;
 517 }
 518
 519 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
 520                                        struct ZebraRecStream *stream,
 521                                        enum zebra_recctrl_action_t action,
 522                                        int test_mode,
 523                                        const char *recordType,
 524                                        zint *sysno,
 525                                        const char *match_criteria,
 526                                        const char *fname,
 527                                        RecType recType,
 528                                        void *recTypeClientData)
 529 {
 530     ZEBRA_RES res = ZEBRA_OK;
 531     while (1)
 532     {
 533         int more = 0;
 534         res = zebra_extract_record_stream(zh, stream,
 535                                           action,
 536                                           test_mode,
 537                                           recordType,
 538                                           sysno,
 539                                           match_criteria,
 540                                           fname,
 541                                           recType, recTypeClientData, &more);
 542         if (!more)
 543         {
 544             res = ZEBRA_OK;
 545             break;
 546         }
 547         if (res != ZEBRA_OK)
 548             break;
 549         if (sysno)
 550             break;
 551     }
 552     return res;
 553 }
 554
 555
 556 ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
 557                                       struct ZebraRecStream *stream,
 558                                       enum zebra_recctrl_action_t action,
 559                                       int test_mode,
 560                                       const char *recordType,
 561                                       zint *sysno,
 562                                       const char *match_criteria,
 563                                       const char *fname,
 564                                       RecType recType,
 565                                       void *recTypeClientData,
 566                                       int *more)
 567
 568 {
 569     zint sysno0 = 0;
 570     RecordAttr *recordAttr;
 571     struct recExtractCtrl extractCtrl;
 572     int r;
 573     const char *matchStr = 0;
 574     Record rec;
 575     off_t start_offset = 0, end_offset = 0;
 576     const char *pr_fname = fname;  /* filename to print .. */
 577     int show_progress = zh->records_processed + zh->records_skipped
 578         < zh->m_file_verbose_limit ? 1:0;
 579
 580     zebra_init_log_level();
 581
 582     if (!pr_fname)
 583         pr_fname = "<no file>";  /* make it printable if file is omitted */
 584
 585     zebra_rec_keys_reset(zh->reg->keys);
 586     zebra_rec_keys_reset(zh->reg->sortKeys);
 587
 588     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
 589     {
 590         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
 591                                       zh->m_explain_database))
 592             return ZEBRA_FAIL;
 593     }
 594
 595     if (stream)
 596     {
 597         off_t null_offset = 0;
 598         extractCtrl.stream = stream;
 599
 600         start_offset = stream->tellf(stream);
 601
 602         extractCtrl.first_record = start_offset ? 0 : 1;
 603
 604         stream->endf(stream, &null_offset);;
 605
 606         extractCtrl.init = extract_init;
 607         extractCtrl.tokenAdd = extract_token_add;
 608         extractCtrl.schemaAdd = extract_schema_add;
 609         extractCtrl.dh = zh->reg->dh;
 610         extractCtrl.handle = zh;
 611         extractCtrl.match_criteria[0] = '\0';
 612         extractCtrl.staticrank = 0;
 613         extractCtrl.action = action;
 614
 615         init_extractCtrl(zh, &extractCtrl);
 616
 617         extract_set_store_data_prepare(&extractCtrl);
 618
 619         r = (*recType->extract)(recTypeClientData, &extractCtrl);
 620
 621         if (action == action_update)
 622         {
 623             action = extractCtrl.action;
 624         }
 625
 626         switch (r)
 627         {
 628         case RECCTRL_EXTRACT_EOF:
 629             return ZEBRA_FAIL;
 630         case RECCTRL_EXTRACT_ERROR_GENERIC:
 631             /* error occured during extraction ... */
 632             yaz_log (YLOG_WARN, "extract error: generic");
 633             return ZEBRA_FAIL;
 634         case RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER:
 635             /* error occured during extraction ... */
 636             yaz_log (YLOG_WARN, "extract error: no such filter");
 637             return ZEBRA_FAIL;
 638         case RECCTRL_EXTRACT_SKIP:
 639             if (show_progress)
 640                 yaz_log (YLOG_LOG, "skip %s %s " ZINT_FORMAT,
 641                          recordType, pr_fname, (zint) start_offset);
 642             *more = 1;
 643
 644             end_offset = stream->endf(stream, 0);
 645             if (end_offset)
 646                 stream->seekf(stream, end_offset);
 647
 648             return ZEBRA_OK;
 649         case RECCTRL_EXTRACT_OK:
 650             break;
 651         default:
 652             yaz_log (YLOG_WARN, "extract error: unknown error: %d", r);
 653             return ZEBRA_FAIL;
 654         }
 655         end_offset = stream->endf(stream, 0);
 656         if (end_offset)
 657             stream->seekf(stream, end_offset);
 658         else
 659             end_offset = stream->tellf(stream);
 660
 661         all_matches_add(&extractCtrl);
 662
 663         if (extractCtrl.match_criteria[0])
 664             match_criteria = extractCtrl.match_criteria;
 665     }
 666
 667     *more = 1;
 668     if (!sysno)
 669     {
 670         sysno = &sysno0;
 671
 672         if (match_criteria && *match_criteria) {
 673             matchStr = match_criteria;
 674         } else {
 675             if (zh->m_record_id && *zh->m_record_id) {
 676                 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
 677                                                zh->m_record_id);
 678                 if (!matchStr)
 679                 {
 680                     yaz_log (YLOG_LOG, "error %s %s " ZINT_FORMAT, recordType,
 681                              pr_fname, (zint) start_offset);
 682                     return ZEBRA_FAIL;
 683                 }
 684             }
 685         }
 686         if (matchStr)
 687         {
 688             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 689             char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
 690                                           matchStr);
 691             if (rinfo)
 692             {
 693                 assert(*rinfo == sizeof(*sysno));
 694                 memcpy (sysno, rinfo+1, sizeof(*sysno));
 695             }
 696         }
 697     }
 698     if (zebra_rec_keys_empty(zh->reg->keys))
 699     {
 700         /* the extraction process returned no information - the record
 701            is probably empty - unless flagShowRecords is in use */
 702         if (test_mode)
 703             return ZEBRA_OK;
 704     }
 705
 706     if (! *sysno)
 707     {
 708         /* new record */
 709         if (action == action_delete)
 710         {
 711             yaz_log (YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 712                          pr_fname, (zint) start_offset);
 713             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
 714             return ZEBRA_FAIL;
 715         }
 716         else if (action == action_replace)
 717         {
 718             yaz_log (YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 719                          pr_fname, (zint) start_offset);
 720             yaz_log (YLOG_WARN, "cannot update record above (seems new)");
 721             return ZEBRA_FAIL;
 722         }
 723         if (show_progress)
 724             yaz_log (YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
 725                      (zint) start_offset);
 726         rec = rec_new (zh->reg->records);
 727
 728         *sysno = rec->sysno;
 729
 730         recordAttr = rec_init_attr (zh->reg->zei, rec);
 731         if (extractCtrl.staticrank < 0)
 732         {
 733             yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0");
 734             extractCtrl.staticrank = 0;
 735         }
 736
 737         if (matchStr)
 738         {
 739             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 740             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
 741                             sizeof(*sysno), sysno);
 742         }
 743
 744         extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 745 #if FLUSH2
 746         extract_flush_record_keys2(zh, *sysno,
 747                                    zh->reg->keys, extractCtrl.staticrank,
 748                                    0, recordAttr->staticrank);
 749 #else
 750         extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 751                                   extractCtrl.staticrank);
 752 #endif
 753         recordAttr->staticrank = extractCtrl.staticrank;
 754         zh->records_inserted++;
 755     }
 756     else
 757     {
 758         /* record already exists */
 759         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 760         zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
 761         if (action == action_insert)
 762         {
 763             yaz_log (YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
 764                          recordType, pr_fname, (zint) start_offset);
 765             logRecord(zh);
 766             return ZEBRA_FAIL;
 767         }
 768
 769         rec = rec_get (zh->reg->records, *sysno);
 770         assert (rec);
 771
 772         recordAttr = rec_init_attr (zh->reg->zei, rec);
 773
 774         /* decrease total size */
 775         zebraExplain_recordBytesIncrement (zh->reg->zei,
 776                                            - recordAttr->recordSize);
 777
 778         zebra_rec_keys_set_buf(delkeys,
 779                                rec->info[recInfo_delKeys],
 780                                rec->size[recInfo_delKeys],
 781                                0);
 782         zebra_rec_keys_set_buf(sortKeys,
 783                                rec->info[recInfo_sortKeys],
 784                                rec->size[recInfo_sortKeys],
 785                                0);
 786
 787         extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
 788 #if !FLUSH2
 789         extract_flush_record_keys(zh, *sysno, 0, delkeys,
 790                                   recordAttr->staticrank);
 791 #endif
 792         if (action == action_delete)
 793         {
 794             /* record going to be deleted */
 795 #if FLUSH2
 796             extract_flush_record_keys2(zh, *sysno, 0, recordAttr->staticrank,
 797                                        delkeys, recordAttr->staticrank);
 798 #endif
 799             if (zebra_rec_keys_empty(delkeys))
 800             {
 801                 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 802                         pr_fname, (zint) start_offset);
 803                 yaz_log(YLOG_WARN, "cannot delete file above, "
 804                         "storeKeys false (3)");
 805             }
 806             else
 807             {
 808                 if (show_progress)
 809                     yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 810                             pr_fname, (zint) start_offset);
 811                 zh->records_deleted++;
 812                 if (matchStr)
 813                 {
 814                     int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 815                     dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
 816                 }
 817                 rec_del (zh->reg->records, &rec);
 818             }
 819             zebra_rec_keys_close(delkeys);
 820             zebra_rec_keys_close(sortKeys);
 821             rec_free(&rec);
 822             logRecord(zh);
 823             return ZEBRA_OK;
 824         }
 825         else
 826         {   /* update or special_update */
 827             if (show_progress)
 828                 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 829                         pr_fname, (zint) start_offset);
 830             extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 831
 832 #if FLUSH2
 833             extract_flush_record_keys2(zh, *sysno,
 834                                        zh->reg->keys, extractCtrl.staticrank,
 835                                        delkeys, recordAttr->staticrank);
 836 #else
 837             extract_flush_record_keys(zh, *sysno, 1,
 838                                       zh->reg->keys, extractCtrl.staticrank);
 839 #endif
 840             recordAttr->staticrank = extractCtrl.staticrank;
 841             zh->records_updated++;
 842         }
 843         zebra_rec_keys_close(delkeys);
 844         zebra_rec_keys_close(sortKeys);
 845     }
 846     /* update file type */
 847     xfree (rec->info[recInfo_fileType]);
 848     rec->info[recInfo_fileType] =
 849         rec_strdup (recordType, &rec->size[recInfo_fileType]);
 850
 851     /* update filename */
 852     xfree (rec->info[recInfo_filename]);
 853     rec->info[recInfo_filename] =
 854         rec_strdup (fname, &rec->size[recInfo_filename]);
 855
 856     /* update delete keys */
 857     xfree (rec->info[recInfo_delKeys]);
 858     if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
 859     {
 860         zebra_rec_keys_get_buf(zh->reg->keys,
 861                                &rec->info[recInfo_delKeys],
 862                                &rec->size[recInfo_delKeys]);
 863     }
 864     else
 865     {
 866         rec->info[recInfo_delKeys] = NULL;
 867         rec->size[recInfo_delKeys] = 0;
 868     }
 869     /* update sort keys */
 870     xfree (rec->info[recInfo_sortKeys]);
 871
 872     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 873                            &rec->info[recInfo_sortKeys],
 874                            &rec->size[recInfo_sortKeys]);
 875
 876     if (stream)
 877     {
 878         recordAttr->recordSize = end_offset - start_offset;
 879         zebraExplain_recordBytesIncrement(zh->reg->zei,
 880                                           recordAttr->recordSize);
 881     }
 882
 883     /* set run-number for this record */
 884     recordAttr->runNumber =
 885         zebraExplain_runNumberIncrement (zh->reg->zei, 0);
 886
 887     /* update store data */
 888     xfree (rec->info[recInfo_storeData]);
 889
 890     /* update store data */
 891     if (zh->store_data_buf)
 892     {
 893         rec->size[recInfo_storeData] = zh->store_data_size;
 894         rec->info[recInfo_storeData] = zh->store_data_buf;
 895         zh->store_data_buf = 0;
 896         recordAttr->recordSize = zh->store_data_size;
 897     }
 898     else if (zh->m_store_data)
 899     {
 900         off_t cur_offset = stream->tellf(stream);
 901
 902         rec->size[recInfo_storeData] = recordAttr->recordSize;
 903         rec->info[recInfo_storeData] = (char *)
 904             xmalloc (recordAttr->recordSize);
 905         stream->seekf(stream, start_offset);
 906         stream->readf(stream, rec->info[recInfo_storeData],
 907                       recordAttr->recordSize);
 908         stream->seekf(stream, cur_offset);
 909     }
 910     else
 911     {
 912         rec->info[recInfo_storeData] = NULL;
 913         rec->size[recInfo_storeData] = 0;
 914     }
 915     /* update database name */
 916     xfree (rec->info[recInfo_databaseName]);
 917     rec->info[recInfo_databaseName] =
 918         rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
 919
 920     /* update offset */
 921     recordAttr->recordOffset = start_offset;
 922
 923     /* commit this record */
 924     rec_put (zh->reg->records, &rec);
 925     logRecord(zh);
 926     return ZEBRA_OK;
 927 }
 928
 929 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
 930 {
 931     ZebraHandle zh = (ZebraHandle) handle;
 932     struct recExtractCtrl extractCtrl;
 933
 934     if (zebraExplain_curDatabase (zh->reg->zei,
 935                                   rec->info[recInfo_databaseName]))
 936     {
 937         abort();
 938         if (zebraExplain_newDatabase (zh->reg->zei,
 939                                       rec->info[recInfo_databaseName], 0))
 940             abort ();
 941     }
 942
 943     zebra_rec_keys_reset(zh->reg->keys);
 944     zebra_rec_keys_reset(zh->reg->sortKeys);
 945
 946     extractCtrl.init = extract_init;
 947     extractCtrl.tokenAdd = extract_token_add;
 948     extractCtrl.schemaAdd = extract_schema_add;
 949     extractCtrl.dh = zh->reg->dh;
 950
 951     init_extractCtrl(zh, &extractCtrl);
 952
 953     extractCtrl.flagShowRecords = 0;
 954     extractCtrl.match_criteria[0] = '\0';
 955     extractCtrl.staticrank = 0;
 956     extractCtrl.action = action_update;
 957
 958     extractCtrl.handle = handle;
 959     extractCtrl.first_record = 1;
 960
 961     extract_set_store_data_prepare(&extractCtrl);
 962
 963     if (n)
 964         grs_extract_tree(&extractCtrl, n);
 965
 966     if (rec->size[recInfo_delKeys])
 967     {
 968         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 969
 970         zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
 971
 972         zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
 973                                rec->size[recInfo_delKeys],
 974                                0);
 975 #if FLUSH2
 976         extract_flush_record_keys2(zh, rec->sysno,
 977                                    zh->reg->keys, 0, delkeys, 0);
 978 #else
 979         extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
 980         extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
 981 #endif
 982         zebra_rec_keys_close(delkeys);
 983
 984         zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
 985                                rec->size[recInfo_sortKeys],
 986                                0);
 987
 988         extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
 989         zebra_rec_keys_close(sortkeys);
 990     }
 991     else
 992     {
 993 #if FLUSH2
 994         extract_flush_record_keys2(zh, rec->sysno, zh->reg->keys, 0, 0, 0);
 995 #else
 996         extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
 997 #endif
 998     }
 999     extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
1000
1001     xfree (rec->info[recInfo_delKeys]);
1002     zebra_rec_keys_get_buf(zh->reg->keys,
1003                            &rec->info[recInfo_delKeys],
1004                            &rec->size[recInfo_delKeys]);
1005
1006     xfree (rec->info[recInfo_sortKeys]);
1007     zebra_rec_keys_get_buf(zh->reg->sortKeys,
1008                            &rec->info[recInfo_sortKeys],
1009                            &rec->size[recInfo_sortKeys]);
1010     return ZEBRA_OK;
1011 }
1012
1013 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
1014                           zebra_rec_keys_t reckeys,
1015                           int level)
1016 {
1017     if (zebra_rec_keys_rewind(reckeys))
1018     {
1019         size_t slen;
1020         const char *str;
1021         struct it_key key;
1022         NMEM nmem = nmem_create();
1023
1024         while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
1025         {
1026             char keystr[200]; /* room for zints to print */
1027             char *dst_term = 0;
1028             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1029             int index_type, i;
1030             const char *string_index;
1031
1032             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1033                                     0/* db */, &string_index);
1034             assert(index_type);
1035             zebra_term_untrans_iconv(zh, nmem, index_type,
1036                                      &dst_term, str);
1037             *keystr = '\0';
1038             for (i = 0; i<key.len; i++)
1039             {
1040                 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key.mem[i]);
1041             }
1042
1043             if (*str < CHR_BASE_CHAR)
1044             {
1045                 int i;
1046                 char dst_buf[200]; /* room for special chars */
1047
1048                 strcpy(dst_buf , "?");
1049
1050                 if (!strcmp(str, ""))
1051                     strcpy(dst_buf, "alwaysmatches");
1052                 if (!strcmp(str, FIRST_IN_FIELD_STR))
1053                     strcpy(dst_buf, "firstinfield");
1054                 else if (!strcmp(str, CHR_UNKNOWN))
1055                     strcpy(dst_buf, "unknown");
1056                 else if (!strcmp(str, CHR_SPACE))
1057                     strcpy(dst_buf, "space");
1058
1059                 for (i = 0; i<slen; i++)
1060                 {
1061                     sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
1062                 }
1063                 yaz_log(level, "%s%c %s %s", keystr, index_type,
1064                         string_index, dst_buf);
1065
1066             }
1067             else
1068                 yaz_log(level, "%s%c %s \"%s\"", keystr, index_type,
1069                         string_index, dst_term);
1070
1071             nmem_reset(nmem);
1072         }
1073         nmem_destroy(nmem);
1074     }
1075 }
1076
1077 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
1078                              zebra_rec_keys_t reckeys)
1079 {
1080     ZebraExplainInfo zei = zh->reg->zei;
1081     struct ord_stat {
1082         int no;
1083         int ord;
1084         struct ord_stat *next;
1085     };
1086
1087     if (zebra_rec_keys_rewind(reckeys))
1088     {
1089         struct ord_stat *ord_list = 0;
1090         struct ord_stat *p;
1091         size_t slen;
1092         const char *str;
1093         struct it_key key_in;
1094         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1095         {
1096             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1097
1098             for (p = ord_list; p ; p = p->next)
1099                 if (p->ord == ord)
1100                 {
1101                     p->no++;
1102                     break;
1103                 }
1104             if (!p)
1105             {
1106                 p = xmalloc(sizeof(*p));
1107                 p->no = 1;
1108                 p->ord = ord;
1109                 p->next = ord_list;
1110                 ord_list = p;
1111             }
1112         }
1113
1114         p = ord_list;
1115         while (p)
1116         {
1117             struct ord_stat *p1 = p;
1118
1119             if (is_insert)
1120                 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
1121             else
1122                 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1123             p = p->next;
1124             xfree(p1);
1125         }
1126     }
1127 }
1128
1129 void extract_flush_record_keys2(ZebraHandle zh, zint sysno,
1130                                 zebra_rec_keys_t ins_keys, zint ins_rank,
1131                                 zebra_rec_keys_t del_keys, zint del_rank)
1132 {
1133     ZebraExplainInfo zei = zh->reg->zei;
1134     int normal = 0;
1135     int optimized = 0;
1136
1137     if (!zh->reg->key_block)
1138     {
1139         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1140         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1141         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1142         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1143     }
1144
1145     if (ins_keys)
1146     {
1147         extract_rec_keys_adjust(zh, 1, ins_keys);
1148         if (!del_keys)
1149             zebraExplain_recordCountIncrement (zei, 1);
1150         zebra_rec_keys_rewind(ins_keys);
1151     }
1152     if (del_keys)
1153     {
1154         extract_rec_keys_adjust(zh, 0, del_keys);
1155         if (!ins_keys)
1156             zebraExplain_recordCountIncrement (zei, -1);
1157         zebra_rec_keys_rewind(del_keys);
1158     }
1159
1160     while (1)
1161     {
1162         size_t del_slen;
1163         const char *del_str;
1164         struct it_key del_key_in;
1165         int del = 0;
1166
1167         size_t ins_slen;
1168         const char *ins_str;
1169         struct it_key ins_key_in;
1170         int ins = 0;
1171
1172         if (del_keys)
1173             del = zebra_rec_keys_read(del_keys, &del_str, &del_slen,
1174                                       &del_key_in);
1175         if (ins_keys)
1176             ins = zebra_rec_keys_read(ins_keys, &ins_str, &ins_slen,
1177                                       &ins_key_in);
1178
1179         if (del && ins && ins_rank == del_rank
1180             && !key_compare(&del_key_in, &ins_key_in)
1181             && ins_slen == del_slen && !memcmp(del_str, ins_str, del_slen))
1182         {
1183             optimized++;
1184             continue;
1185         }
1186         if (!del && !ins)
1187             break;
1188
1189         normal++;
1190         if (del)
1191             key_block_write(zh->reg->key_block, sysno,
1192                             &del_key_in, 0, del_str, del_slen,
1193                             del_rank, zh->m_staticrank);
1194         if (ins)
1195             key_block_write(zh->reg->key_block, sysno,
1196                             &ins_key_in, 1, ins_str, ins_slen,
1197                             ins_rank, zh->m_staticrank);
1198     }
1199     yaz_log(YLOG_LOG, "normal=%d optimized=%d", normal, optimized);
1200 }
1201
1202 void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd,
1203                                zebra_rec_keys_t reckeys,
1204                                zint staticrank)
1205 {
1206     ZebraExplainInfo zei = zh->reg->zei;
1207
1208     extract_rec_keys_adjust(zh, cmd, reckeys);
1209
1210     if (log_level_details)
1211     {
1212         yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1213                 sysno, cmd ? "insert" : "delete");
1214         extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1215     }
1216
1217     if (!zh->reg->key_block)
1218     {
1219         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1220         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1221         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1222         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1223     }
1224     zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1225
1226 #if 0
1227     yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd);
1228     print_rec_keys(zh, reckeys);
1229 #endif
1230     if (zebra_rec_keys_rewind(reckeys))
1231     {
1232         size_t slen;
1233         const char *str;
1234         struct it_key key_in;
1235         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1236         {
1237             key_block_write(zh->reg->key_block, sysno,
1238                             &key_in, cmd, str, slen,
1239                             staticrank, zh->m_staticrank);
1240         }
1241     }
1242 }
1243
1244 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
1245                                      zebra_rec_keys_t reckeys,
1246                                      zebra_snippets *snippets)
1247 {
1248     NMEM nmem = nmem_create();
1249     if (zebra_rec_keys_rewind(reckeys))
1250     {
1251         const char *str;
1252         size_t slen;
1253         struct it_key key;
1254         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1255         {
1256             char *dst_term = 0;
1257             int ord;
1258             zint seqno;
1259             int index_type;
1260
1261             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1262             seqno = key.mem[key.len-1];
1263             ord = CAST_ZINT_TO_INT(key.mem[0]);
1264
1265             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1266                                     0/* db */, 0 /* string_index */);
1267             assert(index_type);
1268             zebra_term_untrans_iconv(zh, nmem, index_type,
1269                                      &dst_term, str);
1270             zebra_snippets_append(snippets, seqno, ord, dst_term);
1271             nmem_reset(nmem);
1272         }
1273     }
1274     nmem_destroy(nmem);
1275     return ZEBRA_OK;
1276 }
1277
1278 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1279 {
1280     yaz_log(YLOG_LOG, "print_rec_keys");
1281     if (zebra_rec_keys_rewind(reckeys))
1282     {
1283         const char *str;
1284         size_t slen;
1285         struct it_key key;
1286         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1287         {
1288             char dst_buf[IT_MAX_WORD];
1289             zint seqno;
1290             int index_type;
1291             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1292             const char *db = 0;
1293             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1294
1295             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1296
1297             seqno = key.mem[key.len-1];
1298
1299             zebra_term_untrans(zh, index_type, dst_buf, str);
1300
1301             yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1302                     " term=%s", ord, seqno, dst_buf);
1303         }
1304     }
1305 }
1306
1307 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1308                                      const char *str, int length)
1309 {
1310     struct it_key key;
1311     ZebraHandle zh = p->extractCtrl->handle;
1312     ZebraExplainInfo zei = zh->reg->zei;
1313     int ch, i;
1314
1315     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1316     if (ch < 0)
1317         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1318
1319     i = 0;
1320     key.mem[i++] = ch;
1321     key.mem[i++] = p->record_id;
1322     key.mem[i++] = p->section_id;
1323
1324     if (zh->m_segment_indexing)
1325         key.mem[i++] = p->segment;
1326     key.mem[i++] = p->seqno;
1327     key.len = i;
1328
1329     zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1330 }
1331
1332 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1333 {
1334     struct it_key key;
1335     ZebraHandle zh = p->extractCtrl->handle;
1336     ZebraExplainInfo zei = zh->reg->zei;
1337     int ch;
1338     zinfo_index_category_t cat = zinfo_index_category_sort;
1339
1340     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1341     if (ch < 0)
1342         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1343     key.len = 2;
1344     key.mem[0] = ch;
1345     key.mem[1] = p->record_id;
1346
1347     zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1348 }
1349
1350 static void extract_add_staticrank_string(RecWord *p,
1351                                           const char *str, int length)
1352 {
1353     char valz[40];
1354     struct recExtractCtrl *ctrl = p->extractCtrl;
1355
1356     if (length > sizeof(valz)-1)
1357         length = sizeof(valz)-1;
1358
1359     memcpy(valz, str, length);
1360     valz[length] = '\0';
1361     ctrl->staticrank = atozint(valz);
1362 }
1363
1364 static void extract_add_string(RecWord *p, const char *string, int length)
1365 {
1366     ZebraHandle zh = p->extractCtrl->handle;
1367     assert (length > 0);
1368
1369     if (!p->index_name)
1370         return;
1371
1372     if (zebra_maps_is_index(zh->reg->zebra_maps, p->index_type))
1373     {
1374         extract_add_index_string(p, zinfo_index_category_index,
1375                                  string, length);
1376         if (zebra_maps_is_alwaysmatches(zh->reg->zebra_maps, p->index_type))
1377         {
1378             RecWord word;
1379             memcpy(&word, p, sizeof(word));
1380
1381             word.seqno = 1;
1382             extract_add_index_string(
1383                 &word, zinfo_index_category_alwaysmatches, "", 0);
1384         }
1385     }
1386     else if (zebra_maps_is_sort(zh->reg->zebra_maps, p->index_type))
1387     {
1388         extract_add_sort_string(p, string, length);
1389     }
1390     else if (zebra_maps_is_staticrank(zh->reg->zebra_maps, p->index_type))
1391     {
1392         extract_add_staticrank_string(p, string, length);
1393     }
1394 }
1395
1396 static void extract_add_incomplete_field(RecWord *p)
1397 {
1398     ZebraHandle zh = p->extractCtrl->handle;
1399     const char *b = p->term_buf;
1400     int remain = p->term_len;
1401     int first = 1;
1402     const char **map = 0;
1403
1404     if (remain > 0)
1405         map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1406
1407     while (map)
1408     {
1409         char buf[IT_MAX_WORD+1];
1410         int i, remain;
1411
1412         /* Skip spaces */
1413         while (map && *map && **map == *CHR_SPACE)
1414         {
1415             remain = p->term_len - (b - p->term_buf);
1416             if (remain > 0)
1417                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b,
1418                                        remain, 0);
1419             else
1420                 map = 0;
1421         }
1422         if (!map)
1423             break;
1424         i = 0;
1425         while (map && *map && **map != *CHR_SPACE)
1426         {
1427             const char *cp = *map;
1428
1429             while (i < IT_MAX_WORD && *cp)
1430                 buf[i++] = *(cp++);
1431             remain = p->term_len - (b - p->term_buf);
1432             if (remain > 0)
1433                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1434             else
1435                 map = 0;
1436         }
1437         if (!i)
1438             return;
1439
1440         if (first)
1441         {
1442             first = 0;
1443             if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type))
1444             {
1445                 /* first in field marker */
1446                 extract_add_string(p, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1447                 p->seqno++;
1448             }
1449         }
1450         extract_add_string (p, buf, i);
1451         p->seqno++;
1452     }
1453 }
1454
1455 static void extract_add_complete_field (RecWord *p)
1456 {
1457     ZebraHandle zh = p->extractCtrl->handle;
1458     const char *b = p->term_buf;
1459     char buf[IT_MAX_WORD+1];
1460     const char **map = 0;
1461     int i = 0, remain = p->term_len;
1462
1463     if (remain > 0)
1464         map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, remain, 1);
1465
1466     while (remain > 0 && i < IT_MAX_WORD)
1467     {
1468         while (map && *map && **map == *CHR_SPACE)
1469         {
1470             remain = p->term_len - (b - p->term_buf);
1471
1472             if (remain > 0)
1473             {
1474                 int first = i ? 0 : 1;  /* first position */
1475                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, first);
1476             }
1477             else
1478                 map = 0;
1479         }
1480         if (!map)
1481             break;
1482
1483         if (i && i < IT_MAX_WORD)
1484             buf[i++] = *CHR_SPACE;
1485         while (map && *map && **map != *CHR_SPACE)
1486         {
1487             const char *cp = *map;
1488
1489             if (**map == *CHR_CUT)
1490             {
1491                 i = 0;
1492             }
1493             else
1494             {
1495                 if (i >= IT_MAX_WORD)
1496                     break;
1497                 while (i < IT_MAX_WORD && *cp)
1498                     buf[i++] = *(cp++);
1499             }
1500             remain = p->term_len  - (b - p->term_buf);
1501             if (remain > 0)
1502             {
1503                 map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b,
1504                                         remain, 0);
1505             }
1506             else
1507                 map = 0;
1508         }
1509     }
1510     if (!i)
1511         return;
1512     extract_add_string (p, buf, i);
1513 }
1514
1515 static void extract_token_add(RecWord *p)
1516 {
1517     ZebraHandle zh = p->extractCtrl->handle;
1518     WRBUF wrbuf;
1519
1520     if (log_level_extract)
1521     {
1522         yaz_log(log_level_extract, "extract_token_add "
1523                 "type=%c index=%s seqno=" ZINT_FORMAT " s=%.*s",
1524                 p->index_type, p->index_name,
1525                 p->seqno, p->term_len, p->term_buf);
1526     }
1527     if ((wrbuf = zebra_replace(zh->reg->zebra_maps, p->index_type, 0,
1528                                p->term_buf, p->term_len)))
1529     {
1530         p->term_buf = wrbuf_buf(wrbuf);
1531         p->term_len = wrbuf_len(wrbuf);
1532     }
1533     if (zebra_maps_is_complete (zh->reg->zebra_maps, p->index_type))
1534         extract_add_complete_field (p);
1535     else
1536         extract_add_incomplete_field(p);
1537 }
1538
1539 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1540                                       void *buf, size_t sz)
1541 {
1542     ZebraHandle zh = (ZebraHandle) p->handle;
1543
1544     xfree(zh->store_data_buf);
1545     zh->store_data_buf = 0;
1546     zh->store_data_size = 0;
1547     if (buf && sz)
1548     {
1549         zh->store_data_buf = xmalloc(sz);
1550         zh->store_data_size = sz;
1551         memcpy(zh->store_data_buf, buf, sz);
1552     }
1553 }
1554
1555 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1556 {
1557     ZebraHandle zh = (ZebraHandle) p->handle;
1558     xfree(zh->store_data_buf);
1559     zh->store_data_buf = 0;
1560     zh->store_data_size = 0;
1561     p->setStoreData = extract_set_store_data_cb;
1562 }
1563
1564 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1565 {
1566     ZebraHandle zh = (ZebraHandle) p->handle;
1567     zebraExplain_addSchema (zh->reg->zei, oid);
1568 }
1569
1570 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1571                              int cmd, zebra_rec_keys_t reckeys)
1572 {
1573 #if 0
1574     yaz_log(YLOG_LOG, "extract_flush_sort_keys cmd=%d sysno=" ZINT_FORMAT,
1575             cmd, sysno);
1576     extract_rec_keys_log(zh, cmd, reckeys, YLOG_LOG);
1577 #endif
1578
1579     if (zebra_rec_keys_rewind(reckeys))
1580     {
1581         zebra_sort_index_t si = zh->reg->sort_index;
1582         size_t slen;
1583         const char *str;
1584         struct it_key key_in;
1585
1586         zebra_sort_sysno(si, sysno);
1587
1588         while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1589         {
1590             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1591
1592             zebra_sort_type(si, ord);
1593             if (cmd == 1)
1594                 zebra_sort_add(si, str, slen);
1595             else
1596                 zebra_sort_delete(si);
1597         }
1598     }
1599 }
1600
1601 /*
1602  * Local variables:
1603  * c-basic-offset: 4
1604  * indent-tabs-mode: nil
1605  * End:
1606  * vim: shiftwidth=4 tabstop=8 expandtab
1607  */
1608