index/extract.c

   1 /* $Id: extract.c,v 1.242 2006-11-25 09:15:19 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #endif
  29 #if HAVE_UNISTD_H
  30 #include <unistd.h>
  31 #endif
  32 #include <fcntl.h>
  33
  34 #include "index.h"
  35 #include "orddict.h"
  36 #include <direntz.h>
  37 #include <charmap.h>
  38
  39 static int log_level_extract = 0;
  40 static int log_level_details = 0;
  41 static int log_level_initialized = 0;
  42
  43 static void zebra_init_log_level(void)
  44 {
  45     if (!log_level_initialized)
  46     {
  47         log_level_initialized = 1;
  48
  49         log_level_extract = yaz_log_module_level("extract");
  50         log_level_details = yaz_log_module_level("indexdetails");
  51     }
  52 }
  53
  54 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
  55                                       int cmd, zebra_rec_keys_t reckeys,
  56                                       zint staticrank);
  57 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
  58                                     int cmd, zebra_rec_keys_t skp);
  59 static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid);
  60 static void extract_token_add (RecWord *p);
  61
  62 static void logRecord (ZebraHandle zh)
  63 {
  64     ++zh->records_processed;
  65     if (!(zh->records_processed % 1000))
  66     {
  67         yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
  68                 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
  69                 zh->records_processed, zh->records_inserted,
  70                 zh->records_updated, zh->records_deleted);
  71     }
  72 }
  73
  74 static void extract_add_index_string (RecWord *p,
  75                                       zinfo_index_category_t cat,
  76                                       const char *str, int length);
  77
  78 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
  79
  80 static void extract_init(struct recExtractCtrl *p, RecWord *w)
  81 {
  82     w->seqno = 1;
  83     w->index_name = "any";
  84     w->index_type = 'w';
  85     w->extractCtrl = p;
  86     w->record_id = 0;
  87     w->section_id = 0;
  88     w->segment = 0;
  89 }
  90
  91 static void searchRecordKey(ZebraHandle zh,
  92                             zebra_rec_keys_t reckeys,
  93                             const char *index_name,
  94                             const char **ws, int ws_length)
  95 {
  96     int i;
  97     int ch = -1;
  98     zinfo_index_category_t cat = zinfo_index_category_index;
  99
 100     for (i = 0; i<ws_length; i++)
 101         ws[i] = NULL;
 102
 103     if (ch < 0)
 104         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, '0', index_name);
 105     if (ch < 0)
 106         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'p', index_name);
 107     if (ch < 0)
 108         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'w', index_name);
 109
 110     if (ch < 0)
 111         return ;
 112
 113     if (zebra_rec_keys_rewind(reckeys))
 114     {
 115         zint startSeq = -1;
 116         const char *str;
 117         size_t slen;
 118         struct it_key key;
 119         zint seqno;
 120         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
 121         {
 122             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
 123
 124             seqno = key.mem[key.len-1];
 125
 126             if (key.mem[0] == ch)
 127             {
 128                 zint woff;
 129
 130                 if (startSeq == -1)
 131                     startSeq = seqno;
 132                 woff = seqno - startSeq;
 133                 if (woff >= 0 && woff < ws_length)
 134                     ws[woff] = str;
 135             }
 136         }
 137     }
 138 }
 139
 140 #define FILE_MATCH_BLANK "\t "
 141
 142 static char *get_match_from_spec(ZebraHandle zh,
 143                           zebra_rec_keys_t reckeys,
 144                           const char *fname, const char *spec)
 145 {
 146     static char dstBuf[2048];      /* static here ??? */
 147     char *dst = dstBuf;
 148     const char *s = spec;
 149
 150     while (1)
 151     {
 152         for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
 153             ;
 154         if (!*s)
 155             break;
 156         if (*s == '(')
 157         {
 158             const char *ws[32];
 159             char attset_str[64], attname_str[64];
 160             int i;
 161             int first = 1;
 162
 163             for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 164                 ;
 165             for (i = 0; *s && *s != ',' && *s != ')' &&
 166                      !strchr(FILE_MATCH_BLANK, *s); s++)
 167                 if (i+1 < sizeof(attset_str))
 168                     attset_str[i++] = *s;
 169             attset_str[i] = '\0';
 170
 171             for (; strchr(FILE_MATCH_BLANK, *s); s++)
 172                 ;
 173             if (*s != ',')
 174                 strcpy(attname_str, attset_str);
 175             else
 176             {
 177                 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 178                     ;
 179                 for (i = 0; *s && *s != ')' &&
 180                          !strchr(FILE_MATCH_BLANK, *s); s++)
 181                     if (i+1 < sizeof(attname_str))
 182                         attname_str[i++] = *s;
 183                 attname_str[i] = '\0';
 184             }
 185
 186             searchRecordKey (zh, reckeys, attname_str, ws, 32);
 187
 188             if (*s != ')')
 189             {
 190                 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
 191                       spec, zh->m_group ? zh->m_group : "none");
 192                 return NULL;
 193             }
 194             s++;
 195
 196             for (i = 0; i<32; i++)
 197                 if (ws[i])
 198                 {
 199                     if (first)
 200                     {
 201                         *dst++ = ' ';
 202                         first = 0;
 203                     }
 204                     strcpy (dst, ws[i]);
 205                     dst += strlen(ws[i]);
 206                 }
 207             if (first)
 208             {
 209                 yaz_log (YLOG_WARN, "Record didn't contain match"
 210                       " fields in (%s,%s)", attset_str, attname_str);
 211                 return NULL;
 212             }
 213         }
 214         else if (*s == '$')
 215         {
 216             int spec_len;
 217             char special[64];
 218             const char *spec_src = NULL;
 219             const char *s1 = ++s;
 220             while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
 221                 s1++;
 222
 223             spec_len = s1 - s;
 224             if (spec_len > sizeof(special)-1)
 225                 spec_len = sizeof(special)-1;
 226             memcpy (special, s, spec_len);
 227             special[spec_len] = '\0';
 228             s = s1;
 229
 230             if (!strcmp (special, "group"))
 231                 spec_src = zh->m_group;
 232             else if (!strcmp (special, "database"))
 233                 spec_src = zh->basenames[0];
 234             else if (!strcmp (special, "filename")) {
 235                 spec_src = fname;
 236             }
 237             else if (!strcmp (special, "type"))
 238                 spec_src = zh->m_record_type;
 239             else
 240                 spec_src = NULL;
 241             if (spec_src)
 242             {
 243                 strcpy (dst, spec_src);
 244                 dst += strlen (spec_src);
 245             }
 246         }
 247         else if (*s == '\"' || *s == '\'')
 248         {
 249             int stopMarker = *s++;
 250             char tmpString[64];
 251             int i = 0;
 252
 253             while (*s && *s != stopMarker)
 254             {
 255                 if (i+1 < sizeof(tmpString))
 256                     tmpString[i++] = *s++;
 257             }
 258             if (*s)
 259                 s++;
 260             tmpString[i] = '\0';
 261             strcpy (dst, tmpString);
 262             dst += strlen (tmpString);
 263         }
 264         else
 265         {
 266             yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
 267                   spec, zh->m_group ? zh->m_group : "none");
 268             return NULL;
 269         }
 270         *dst++ = 1;
 271     }
 272     if (dst == dstBuf)
 273     {
 274         yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
 275               fname, zh->m_group ? zh->m_group : "none");
 276         return NULL;
 277     }
 278     *dst = '\0';
 279     return dstBuf;
 280 }
 281
 282 struct recordLogInfo {
 283     const char *fname;
 284     int recordOffset;
 285     struct recordGroup *rGroup;
 286 };
 287
 288 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
 289 {
 290     int i;
 291     for (i = 0; i<256; i++)
 292     {
 293         if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
 294             ctrl->seqno[i] = 1;
 295         else
 296             ctrl->seqno[i] = 0;
 297     }
 298     ctrl->flagShowRecords = !zh->m_flag_rw;
 299 }
 300
 301 static void all_matches_add(struct recExtractCtrl *ctrl)
 302 {
 303     RecWord word;
 304     extract_init(ctrl, &word);
 305     word.index_name = "_ALLRECORDS";
 306     word.index_type = 'w';
 307     word.seqno = 1;
 308     extract_add_index_string (&word, zinfo_index_category_alwaysmatches,
 309                               "", 0);
 310 }
 311
 312 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
 313                              int deleteFlag)
 314 {
 315     ZEBRA_RES r = ZEBRA_OK;
 316     int i, fd;
 317     char gprefix[128];
 318     char ext[128];
 319     char ext_res[128];
 320     struct file_read_info *fi = 0;
 321     const char *original_record_type = 0;
 322     RecType recType;
 323     void *recTypeClientData;
 324     struct ZebraRecStream stream, *streamp;
 325
 326     zebra_init_log_level();
 327
 328     if (!zh->m_group || !*zh->m_group)
 329         *gprefix = '\0';
 330     else
 331         sprintf (gprefix, "%s.", zh->m_group);
 332
 333     yaz_log(log_level_extract, "zebra_extract_file %s", fname);
 334
 335     /* determine file extension */
 336     *ext = '\0';
 337     for (i = strlen(fname); --i >= 0; )
 338         if (fname[i] == '/')
 339             break;
 340         else if (fname[i] == '.')
 341         {
 342             strcpy (ext, fname+i+1);
 343             break;
 344         }
 345     /* determine file type - depending on extension */
 346     original_record_type = zh->m_record_type;
 347     if (!zh->m_record_type)
 348     {
 349         sprintf (ext_res, "%srecordType.%s", gprefix, ext);
 350         zh->m_record_type = res_get (zh->res, ext_res);
 351     }
 352     if (!zh->m_record_type)
 353     {
 354         if (zh->records_processed < zh->m_file_verbose_limit)
 355             yaz_log (YLOG_LOG, "? %s", fname);
 356         return 0;
 357     }
 358     /* determine match criteria */
 359     if (!zh->m_record_id)
 360     {
 361         sprintf (ext_res, "%srecordId.%s", gprefix, ext);
 362         zh->m_record_id = res_get (zh->res, ext_res);
 363     }
 364
 365     if (!(recType =
 366           recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
 367                           &recTypeClientData)))
 368     {
 369         yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
 370         return ZEBRA_FAIL;
 371     }
 372
 373     switch(recType->version)
 374     {
 375     case 0:
 376         break;
 377     default:
 378         yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
 379     }
 380     if (sysno && deleteFlag)
 381     {
 382         streamp = 0;
 383         fi = 0;
 384     }
 385     else
 386     {
 387         char full_rep[1024];
 388
 389         if (zh->path_reg && !yaz_is_abspath (fname))
 390         {
 391             strcpy (full_rep, zh->path_reg);
 392             strcat (full_rep, "/");
 393             strcat (full_rep, fname);
 394         }
 395         else
 396             strcpy (full_rep, fname);
 397
 398         if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
 399         {
 400             yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
 401             zh->m_record_type = original_record_type;
 402             return ZEBRA_FAIL;
 403         }
 404         streamp = &stream;
 405         zebra_create_stream_fd(streamp, fd, 0);
 406     }
 407     while(1)
 408     {
 409         int more = 0;
 410         r = zebra_extract_record_stream(zh, streamp,
 411                                         deleteFlag,
 412                                         0, /* tst_mode */
 413                                         zh->m_record_type,
 414                                         sysno,
 415                                         0, /*match_criteria */
 416                                         fname,
 417                                         1, /* force_update */
 418                                         1, /* allow_update */
 419                                         recType, recTypeClientData, &more);
 420         if (!more)
 421             break;
 422         if (sysno)
 423         {
 424             break;
 425         }
 426     }
 427     if (streamp)
 428         stream.destroy(streamp);
 429     zh->m_record_type = original_record_type;
 430     return r;
 431 }
 432
 433 /*
 434   If sysno is provided, then it's used to identify the reocord.
 435   If not, and match_criteria is provided, then sysno is guessed
 436   If not, and a record is provided, then sysno is got from there
 437
 438  */
 439
 440 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
 441                                       const char *buf, size_t buf_size,
 442                                       int delete_flag,
 443                                       int test_mode,
 444                                       const char *recordType,
 445                                       zint *sysno,
 446                                       const char *match_criteria,
 447                                       const char *fname,
 448                                       int force_update,
 449                                       int allow_update)
 450 {
 451     struct ZebraRecStream stream;
 452     ZEBRA_RES res;
 453     void *clientData;
 454     RecType recType = 0;
 455     int more = 0;
 456
 457     if (recordType && *recordType)
 458     {
 459         yaz_log(log_level_extract,
 460                 "Record type explicitly specified: %s", recordType);
 461         recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
 462                                   &clientData);
 463     }
 464     else
 465     {
 466         if (!(zh->m_record_type))
 467         {
 468             yaz_log (YLOG_WARN, "No such record type defined");
 469             return ZEBRA_FAIL;
 470         }
 471         yaz_log(log_level_extract, "Get record type from rgroup: %s",
 472                 zh->m_record_type);
 473         recType = recType_byName (zh->reg->recTypes, zh->res,
 474                                   zh->m_record_type, &clientData);
 475         recordType = zh->m_record_type;
 476     }
 477
 478     if (!recType)
 479     {
 480         yaz_log (YLOG_WARN, "No such record type: %s", recordType);
 481         return ZEBRA_FAIL;
 482     }
 483
 484     zebra_create_stream_mem(&stream, buf, buf_size);
 485
 486     res = zebra_extract_record_stream(zh, &stream,
 487                                       delete_flag,
 488                                       test_mode,
 489                                       recordType,
 490                                       sysno,
 491                                       match_criteria,
 492                                       fname,
 493                                       force_update,
 494                                       allow_update,
 495                                       recType, clientData, &more);
 496     stream.destroy(&stream);
 497     return res;
 498 }
 499
 500
 501 ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
 502                                       struct ZebraRecStream *stream,
 503                                       int delete_flag,
 504                                       int test_mode,
 505                                       const char *recordType,
 506                                       zint *sysno,
 507                                       const char *match_criteria,
 508                                       const char *fname,
 509                                       int force_update,
 510                                       int allow_update,
 511                                       RecType recType,
 512                                       void *recTypeClientData,
 513                                       int *more)
 514
 515 {
 516     zint sysno0 = 0;
 517     RecordAttr *recordAttr;
 518     struct recExtractCtrl extractCtrl;
 519     int r;
 520     const char *matchStr = 0;
 521     Record rec;
 522     off_t start_offset = 0, end_offset = 0;
 523     const char *pr_fname = fname;  /* filename to print .. */
 524     int show_progress = zh->records_processed < zh->m_file_verbose_limit ? 1:0;
 525
 526     zebra_init_log_level();
 527
 528     if (!pr_fname)
 529         pr_fname = "<no file>";  /* make it printable if file is omitted */
 530
 531     zebra_rec_keys_reset(zh->reg->keys);
 532     zebra_rec_keys_reset(zh->reg->sortKeys);
 533
 534     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
 535     {
 536         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
 537                                       zh->m_explain_database))
 538             return ZEBRA_FAIL;
 539     }
 540
 541     if (stream)
 542     {
 543         off_t null_offset = 0;
 544         extractCtrl.stream = stream;
 545
 546         start_offset = stream->tellf(stream);
 547
 548         extractCtrl.first_record = start_offset ? 0 : 1;
 549
 550         stream->endf(stream, &null_offset);;
 551
 552         extractCtrl.init = extract_init;
 553         extractCtrl.tokenAdd = extract_token_add;
 554         extractCtrl.schemaAdd = extract_schema_add;
 555         extractCtrl.dh = zh->reg->dh;
 556         extractCtrl.handle = zh;
 557         extractCtrl.match_criteria[0] = '\0';
 558         extractCtrl.staticrank = 0;
 559
 560         init_extractCtrl(zh, &extractCtrl);
 561
 562         extract_set_store_data_prepare(&extractCtrl);
 563
 564         r = (*recType->extract)(recTypeClientData, &extractCtrl);
 565
 566         if (r == RECCTRL_EXTRACT_EOF)
 567             return ZEBRA_FAIL;
 568         else if (r == RECCTRL_EXTRACT_ERROR_GENERIC)
 569         {
 570             /* error occured during extraction ... */
 571             yaz_log (YLOG_WARN, "extract error: generic");
 572             return ZEBRA_FAIL;
 573         }
 574         else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER)
 575         {
 576             /* error occured during extraction ... */
 577             yaz_log (YLOG_WARN, "extract error: no such filter");
 578             return ZEBRA_FAIL;
 579         }
 580
 581         all_matches_add(&extractCtrl);
 582
 583         if (extractCtrl.match_criteria[0])
 584             match_criteria = extractCtrl.match_criteria;
 585
 586
 587         end_offset = stream->endf(stream, 0);
 588
 589         if (!end_offset)
 590             end_offset = stream->tellf(stream);
 591         else
 592             stream->seekf(stream, end_offset);
 593
 594     }
 595
 596
 597     *more = 1;
 598     if (!sysno)
 599     {
 600         sysno = &sysno0;
 601
 602         if (match_criteria && *match_criteria) {
 603             matchStr = match_criteria;
 604         } else {
 605             if (zh->m_record_id && *zh->m_record_id) {
 606                 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
 607                                                zh->m_record_id);
 608                 if (!matchStr)
 609                 {
 610                     yaz_log (YLOG_LOG, "error %s %s " ZINT_FORMAT, recordType,
 611                              pr_fname, (zint) start_offset);
 612                     return ZEBRA_FAIL;
 613                 }
 614             }
 615         }
 616         if (matchStr)
 617         {
 618             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 619             char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
 620                                           matchStr);
 621             if (rinfo)
 622             {
 623                 assert(*rinfo == sizeof(*sysno));
 624                 memcpy (sysno, rinfo+1, sizeof(*sysno));
 625             }
 626         }
 627     }
 628     if (zebra_rec_keys_empty(zh->reg->keys))
 629     {
 630         /* the extraction process returned no information - the record
 631            is probably empty - unless flagShowRecords is in use */
 632         if (test_mode)
 633             return ZEBRA_OK;
 634     }
 635
 636     if (! *sysno)
 637     {
 638         /* new record */
 639         if (delete_flag)
 640         {
 641             yaz_log (YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 642                          pr_fname, (zint) start_offset);
 643             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
 644             return ZEBRA_FAIL;
 645         }
 646         if (show_progress)
 647             yaz_log (YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
 648                      (zint) start_offset);
 649         rec = rec_new (zh->reg->records);
 650
 651         *sysno = rec->sysno;
 652
 653         recordAttr = rec_init_attr (zh->reg->zei, rec);
 654         recordAttr->staticrank = extractCtrl.staticrank;
 655
 656         if (matchStr)
 657         {
 658             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 659             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
 660                             sizeof(*sysno), sysno);
 661         }
 662
 663         extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 664         extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 665                                   recordAttr->staticrank);
 666         zh->records_inserted++;
 667     }
 668     else
 669     {
 670         /* record already exists */
 671         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 672         zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
 673         if (!allow_update)
 674         {
 675             yaz_log (YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
 676                          recordType, pr_fname, (zint) start_offset);
 677             logRecord(zh);
 678             return ZEBRA_FAIL;
 679         }
 680
 681         rec = rec_get (zh->reg->records, *sysno);
 682         assert (rec);
 683
 684         recordAttr = rec_init_attr (zh->reg->zei, rec);
 685
 686         /* decrease total size */
 687         zebraExplain_recordBytesIncrement (zh->reg->zei,
 688                                            - recordAttr->recordSize);
 689
 690         zebra_rec_keys_set_buf(delkeys,
 691                                rec->info[recInfo_delKeys],
 692                                rec->size[recInfo_delKeys],
 693                                0);
 694         zebra_rec_keys_set_buf(sortKeys,
 695                                rec->info[recInfo_sortKeys],
 696                                rec->size[recInfo_sortKeys],
 697                                0);
 698
 699         extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
 700         extract_flush_record_keys(zh, *sysno, 0, delkeys,
 701                                   recordAttr->staticrank);
 702         if (delete_flag)
 703         {
 704             /* record going to be deleted */
 705             if (zebra_rec_keys_empty(delkeys))
 706             {
 707                 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 708                         pr_fname, (zint) start_offset);
 709                 yaz_log(YLOG_WARN, "cannot delete file above, "
 710                         "storeKeys false (3)");
 711             }
 712             else
 713             {
 714                 if (show_progress)
 715                     yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 716                             pr_fname, (zint) start_offset);
 717                 zh->records_deleted++;
 718                 if (matchStr)
 719                 {
 720                     int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 721                     dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
 722                 }
 723                 rec_del (zh->reg->records, &rec);
 724             }
 725             rec_free(&rec);
 726             logRecord(zh);
 727             return ZEBRA_OK;
 728         }
 729         else
 730         {
 731             if (show_progress)
 732                 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 733                         pr_fname, (zint) start_offset);
 734             recordAttr->staticrank = extractCtrl.staticrank;
 735             extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 736             extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 737                                       recordAttr->staticrank);
 738             zh->records_updated++;
 739         }
 740         zebra_rec_keys_close(delkeys);
 741         zebra_rec_keys_close(sortKeys);
 742     }
 743     /* update file type */
 744     xfree (rec->info[recInfo_fileType]);
 745     rec->info[recInfo_fileType] =
 746         rec_strdup (recordType, &rec->size[recInfo_fileType]);
 747
 748     /* update filename */
 749     xfree (rec->info[recInfo_filename]);
 750     rec->info[recInfo_filename] =
 751         rec_strdup (fname, &rec->size[recInfo_filename]);
 752
 753     /* update delete keys */
 754     xfree (rec->info[recInfo_delKeys]);
 755     if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
 756     {
 757         zebra_rec_keys_get_buf(zh->reg->keys,
 758                                &rec->info[recInfo_delKeys],
 759                                &rec->size[recInfo_delKeys]);
 760     }
 761     else
 762     {
 763         rec->info[recInfo_delKeys] = NULL;
 764         rec->size[recInfo_delKeys] = 0;
 765     }
 766     /* update sort keys */
 767     xfree (rec->info[recInfo_sortKeys]);
 768
 769     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 770                            &rec->info[recInfo_sortKeys],
 771                            &rec->size[recInfo_sortKeys]);
 772
 773     if (stream)
 774     {
 775         recordAttr->recordSize = end_offset - start_offset;
 776         zebraExplain_recordBytesIncrement(zh->reg->zei,
 777                                           recordAttr->recordSize);
 778     }
 779
 780     /* set run-number for this record */
 781     recordAttr->runNumber =
 782         zebraExplain_runNumberIncrement (zh->reg->zei, 0);
 783
 784     /* update store data */
 785     xfree (rec->info[recInfo_storeData]);
 786
 787     /* update store data */
 788     if (zh->store_data_buf)
 789     {
 790         rec->size[recInfo_storeData] = zh->store_data_size;
 791         rec->info[recInfo_storeData] = zh->store_data_buf;
 792         zh->store_data_buf = 0;
 793         recordAttr->recordSize = zh->store_data_size;
 794     }
 795     else if (zh->m_store_data)
 796     {
 797         off_t cur_offset = stream->tellf(stream);
 798
 799         rec->size[recInfo_storeData] = recordAttr->recordSize;
 800         rec->info[recInfo_storeData] = (char *)
 801             xmalloc (recordAttr->recordSize);
 802         stream->seekf(stream, start_offset);
 803         stream->readf(stream, rec->info[recInfo_storeData],
 804                       recordAttr->recordSize);
 805         stream->seekf(stream, cur_offset);
 806     }
 807     else
 808     {
 809         rec->info[recInfo_storeData] = NULL;
 810         rec->size[recInfo_storeData] = 0;
 811     }
 812     /* update database name */
 813     xfree (rec->info[recInfo_databaseName]);
 814     rec->info[recInfo_databaseName] =
 815         rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
 816
 817     /* update offset */
 818     recordAttr->recordOffset = start_offset;
 819
 820     /* commit this record */
 821     rec_put (zh->reg->records, &rec);
 822     logRecord(zh);
 823     return ZEBRA_OK;
 824 }
 825
 826 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
 827 {
 828     ZebraHandle zh = (ZebraHandle) handle;
 829     struct recExtractCtrl extractCtrl;
 830
 831     if (zebraExplain_curDatabase (zh->reg->zei,
 832                                   rec->info[recInfo_databaseName]))
 833     {
 834         abort();
 835         if (zebraExplain_newDatabase (zh->reg->zei,
 836                                       rec->info[recInfo_databaseName], 0))
 837             abort ();
 838     }
 839
 840     zebra_rec_keys_reset(zh->reg->keys);
 841     zebra_rec_keys_reset(zh->reg->sortKeys);
 842
 843     extractCtrl.init = extract_init;
 844     extractCtrl.tokenAdd = extract_token_add;
 845     extractCtrl.schemaAdd = extract_schema_add;
 846     extractCtrl.dh = zh->reg->dh;
 847
 848     init_extractCtrl(zh, &extractCtrl);
 849
 850     extractCtrl.flagShowRecords = 0;
 851     extractCtrl.match_criteria[0] = '\0';
 852     extractCtrl.staticrank = 0;
 853     extractCtrl.handle = handle;
 854     extractCtrl.first_record = 1;
 855
 856     extract_set_store_data_prepare(&extractCtrl);
 857
 858     if (n)
 859         grs_extract_tree(&extractCtrl, n);
 860
 861     if (rec->size[recInfo_delKeys])
 862     {
 863         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 864
 865         zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
 866
 867         zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
 868                                rec->size[recInfo_delKeys],
 869                                0);
 870         extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
 871         zebra_rec_keys_close(delkeys);
 872
 873         zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
 874                                rec->size[recInfo_sortKeys],
 875                                0);
 876
 877         extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
 878         zebra_rec_keys_close(sortkeys);
 879     }
 880     extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
 881     extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
 882
 883     xfree (rec->info[recInfo_delKeys]);
 884     zebra_rec_keys_get_buf(zh->reg->keys,
 885                            &rec->info[recInfo_delKeys],
 886                            &rec->size[recInfo_delKeys]);
 887
 888     xfree (rec->info[recInfo_sortKeys]);
 889     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 890                            &rec->info[recInfo_sortKeys],
 891                            &rec->size[recInfo_sortKeys]);
 892     return ZEBRA_OK;
 893 }
 894
 895 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
 896                           zebra_rec_keys_t reckeys,
 897                           int level)
 898 {
 899     if (zebra_rec_keys_rewind(reckeys))
 900     {
 901         size_t slen;
 902         const char *str;
 903         struct it_key key;
 904         NMEM nmem = nmem_create();
 905
 906         while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
 907         {
 908             char keystr[200]; /* room for zints to print */
 909             char *dst_term = 0;
 910             int ord = CAST_ZINT_TO_INT(key.mem[0]);
 911             int index_type, i;
 912             const char *string_index;
 913
 914             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
 915                                     0/* db */, &string_index);
 916             assert(index_type);
 917             zebra_term_untrans_iconv(zh, nmem, index_type,
 918                                      &dst_term, str);
 919             *keystr = '\0';
 920             for (i = 0; i<key.len; i++)
 921             {
 922                 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key.mem[i]);
 923             }
 924
 925             if (*str < CHR_BASE_CHAR)
 926             {
 927                 int i;
 928                 char dst_buf[200]; /* room for special chars */
 929
 930                 strcpy(dst_buf , "?");
 931
 932                 if (!strcmp(str, ""))
 933                     strcpy(dst_buf, "alwaysmatches");
 934                 if (!strcmp(str, FIRST_IN_FIELD_STR))
 935                     strcpy(dst_buf, "firstinfield");
 936                 else if (!strcmp(str, CHR_UNKNOWN))
 937                     strcpy(dst_buf, "unknown");
 938                 else if (!strcmp(str, CHR_SPACE))
 939                     strcpy(dst_buf, "space");
 940
 941                 for (i = 0; i<slen; i++)
 942                 {
 943                     sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
 944                 }
 945                 yaz_log(level, "%s%c %s %s", keystr, index_type,
 946                         string_index, dst_buf);
 947
 948             }
 949             else
 950                 yaz_log(level, "%s%c %s \"%s\"", keystr, index_type,
 951                         string_index, dst_term);
 952
 953             nmem_reset(nmem);
 954         }
 955         nmem_destroy(nmem);
 956     }
 957 }
 958
 959 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
 960                              zebra_rec_keys_t reckeys)
 961 {
 962     ZebraExplainInfo zei = zh->reg->zei;
 963     struct ord_stat {
 964         int no;
 965         int ord;
 966         struct ord_stat *next;
 967     };
 968
 969     if (zebra_rec_keys_rewind(reckeys))
 970     {
 971         struct ord_stat *ord_list = 0;
 972         struct ord_stat *p;
 973         size_t slen;
 974         const char *str;
 975         struct it_key key_in;
 976         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
 977         {
 978             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
 979
 980             for (p = ord_list; p ; p = p->next)
 981                 if (p->ord == ord)
 982                 {
 983                     p->no++;
 984                     break;
 985                 }
 986             if (!p)
 987             {
 988                 p = xmalloc(sizeof(*p));
 989                 p->no = 1;
 990                 p->ord = ord;
 991                 p->next = ord_list;
 992                 ord_list = p;
 993             }
 994         }
 995
 996         p = ord_list;
 997         while (p)
 998         {
 999             struct ord_stat *p1 = p;
1000
1001             if (is_insert)
1002                 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
1003             else
1004                 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1005             p = p->next;
1006             xfree(p1);
1007         }
1008     }
1009 }
1010
1011 void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd,
1012                                zebra_rec_keys_t reckeys,
1013                                zint staticrank)
1014 {
1015     ZebraExplainInfo zei = zh->reg->zei;
1016
1017     extract_rec_keys_adjust(zh, cmd, reckeys);
1018
1019     if (log_level_details)
1020     {
1021         yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1022                 sysno, cmd ? "insert" : "delete");
1023         extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1024     }
1025
1026     if (!zh->reg->key_block)
1027     {
1028         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1029         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1030         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1031         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1032     }
1033     zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1034
1035     if (zebra_rec_keys_rewind(reckeys))
1036     {
1037         size_t slen;
1038         const char *str;
1039         struct it_key key_in;
1040         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1041         {
1042             key_block_write(zh->reg->key_block, sysno,
1043                             &key_in, cmd, str, slen,
1044                             staticrank, zh->m_staticrank);
1045         }
1046     }
1047 }
1048
1049
1050 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
1051                                      zebra_rec_keys_t reckeys,
1052                                      zebra_snippets *snippets)
1053 {
1054     NMEM nmem = nmem_create();
1055     if (zebra_rec_keys_rewind(reckeys))
1056     {
1057         const char *str;
1058         size_t slen;
1059         struct it_key key;
1060         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1061         {
1062             char dst_buf[IT_MAX_WORD];
1063             char *dst_term = dst_buf;
1064             int ord;
1065             zint seqno;
1066             int index_type;
1067
1068             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1069             seqno = key.mem[key.len-1];
1070             ord = CAST_ZINT_TO_INT(key.mem[0]);
1071
1072             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1073                                     0/* db */, 0 /* string_index */);
1074             assert(index_type);
1075             zebra_term_untrans_iconv(zh, nmem, index_type,
1076                                      &dst_term, str);
1077             zebra_snippets_append(snippets, seqno, ord, dst_term);
1078             nmem_reset(nmem);
1079         }
1080     }
1081     nmem_destroy(nmem);
1082     return ZEBRA_OK;
1083 }
1084
1085 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1086 {
1087     yaz_log(YLOG_LOG, "print_rec_keys");
1088     if (zebra_rec_keys_rewind(reckeys))
1089     {
1090         const char *str;
1091         size_t slen;
1092         struct it_key key;
1093         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1094         {
1095             char dst_buf[IT_MAX_WORD];
1096             zint seqno;
1097             int index_type;
1098             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1099             const char *db = 0;
1100             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1101
1102             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1103
1104             seqno = key.mem[key.len-1];
1105
1106             zebra_term_untrans(zh, index_type, dst_buf, str);
1107
1108             yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1109                     " term=%s", ord, seqno, dst_buf);
1110         }
1111     }
1112 }
1113
1114 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1115                                      const char *str, int length)
1116 {
1117     struct it_key key;
1118     ZebraHandle zh = p->extractCtrl->handle;
1119     ZebraExplainInfo zei = zh->reg->zei;
1120     int ch, i;
1121
1122     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1123     if (ch < 0)
1124         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1125
1126     i = 0;
1127     key.mem[i++] = ch;
1128     key.mem[i++] = p->record_id;
1129     key.mem[i++] = p->section_id;
1130
1131     if (zh->m_segment_indexing)
1132         key.mem[i++] = p->segment;
1133     key.mem[i++] = p->seqno;
1134     key.len = i;
1135
1136     zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1137 }
1138
1139 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1140 {
1141     struct it_key key;
1142     ZebraHandle zh = p->extractCtrl->handle;
1143     ZebraExplainInfo zei = zh->reg->zei;
1144     int ch;
1145     zinfo_index_category_t cat = zinfo_index_category_sort;
1146
1147     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1148     if (ch < 0)
1149         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1150     key.len = 2;
1151     key.mem[0] = ch;
1152     key.mem[1] = p->record_id;
1153
1154     zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1155 }
1156
1157 static void extract_add_string(RecWord *p, const char *string, int length)
1158 {
1159     ZebraHandle zh = p->extractCtrl->handle;
1160     assert (length > 0);
1161
1162     if (!p->index_name)
1163         return;
1164
1165     if (zebra_maps_is_sort(zh->reg->zebra_maps, p->index_type))
1166         extract_add_sort_string(p, string, length);
1167     else
1168     {
1169         extract_add_index_string(p, zinfo_index_category_index,
1170                                  string, length);
1171         if (zebra_maps_is_alwaysmatches(zh->reg->zebra_maps, p->index_type))
1172         {
1173             RecWord word;
1174             memcpy(&word, p, sizeof(word));
1175
1176             word.seqno = 1;
1177             extract_add_index_string(
1178                 &word, zinfo_index_category_alwaysmatches, "", 0);
1179         }
1180     }
1181 }
1182
1183 static void extract_add_incomplete_field(RecWord *p)
1184 {
1185     ZebraHandle zh = p->extractCtrl->handle;
1186     const char *b = p->term_buf;
1187     int remain = p->term_len;
1188     const char **map = 0;
1189
1190     if (remain > 0)
1191         map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1192
1193     if (map)
1194     {
1195         if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type))
1196         {
1197              /* first in field marker */
1198             extract_add_string(p, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1199             p->seqno++;
1200         }
1201     }
1202     while (map)
1203     {
1204         char buf[IT_MAX_WORD+1];
1205         int i, remain;
1206
1207         /* Skip spaces */
1208         while (map && *map && **map == *CHR_SPACE)
1209         {
1210             remain = p->term_len - (b - p->term_buf);
1211             if (remain > 0)
1212                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b,
1213                                        remain, 0);
1214             else
1215                 map = 0;
1216         }
1217         if (!map)
1218             break;
1219         i = 0;
1220         while (map && *map && **map != *CHR_SPACE)
1221         {
1222             const char *cp = *map;
1223
1224             while (i < IT_MAX_WORD && *cp)
1225                 buf[i++] = *(cp++);
1226             remain = p->term_len - (b - p->term_buf);
1227             if (remain > 0)
1228                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1229             else
1230                 map = 0;
1231         }
1232         if (!i)
1233             return;
1234         extract_add_string (p, buf, i);
1235         p->seqno++;
1236     }
1237 }
1238
1239 static void extract_add_complete_field (RecWord *p)
1240 {
1241     ZebraHandle zh = p->extractCtrl->handle;
1242     const char *b = p->term_buf;
1243     char buf[IT_MAX_WORD+1];
1244     const char **map = 0;
1245     int i = 0, remain = p->term_len;
1246
1247     if (remain > 0)
1248         map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, remain, 1);
1249
1250     while (remain > 0 && i < IT_MAX_WORD)
1251     {
1252         while (map && *map && **map == *CHR_SPACE)
1253         {
1254             remain = p->term_len - (b - p->term_buf);
1255
1256             if (remain > 0)
1257             {
1258                 int first = i ? 0 : 1;  /* first position */
1259                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, first);
1260             }
1261             else
1262                 map = 0;
1263         }
1264         if (!map)
1265             break;
1266
1267         if (i && i < IT_MAX_WORD)
1268             buf[i++] = *CHR_SPACE;
1269         while (map && *map && **map != *CHR_SPACE)
1270         {
1271             const char *cp = *map;
1272
1273             if (**map == *CHR_CUT)
1274             {
1275                 i = 0;
1276             }
1277             else
1278             {
1279                 if (i >= IT_MAX_WORD)
1280                     break;
1281                 while (i < IT_MAX_WORD && *cp)
1282                     buf[i++] = *(cp++);
1283             }
1284             remain = p->term_len  - (b - p->term_buf);
1285             if (remain > 0)
1286             {
1287                 map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b,
1288                                         remain, 0);
1289             }
1290             else
1291                 map = 0;
1292         }
1293     }
1294     if (!i)
1295         return;
1296     extract_add_string (p, buf, i);
1297 }
1298
1299 static void extract_token_add(RecWord *p)
1300 {
1301     ZebraHandle zh = p->extractCtrl->handle;
1302     WRBUF wrbuf;
1303
1304     if (log_level_extract)
1305     {
1306         yaz_log(log_level_extract, "extract_token_add "
1307                 "type=%c index=%s seqno=" ZINT_FORMAT " s=%.*s",
1308                 p->index_type, p->index_name,
1309                 p->seqno, p->term_len, p->term_buf);
1310     }
1311     if ((wrbuf = zebra_replace(zh->reg->zebra_maps, p->index_type, 0,
1312                                p->term_buf, p->term_len)))
1313     {
1314         p->term_buf = wrbuf_buf(wrbuf);
1315         p->term_len = wrbuf_len(wrbuf);
1316     }
1317     if (zebra_maps_is_complete (zh->reg->zebra_maps, p->index_type))
1318         extract_add_complete_field (p);
1319     else
1320         extract_add_incomplete_field(p);
1321 }
1322
1323 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1324                                       void *buf, size_t sz)
1325 {
1326     ZebraHandle zh = (ZebraHandle) p->handle;
1327
1328     xfree(zh->store_data_buf);
1329     zh->store_data_buf = 0;
1330     zh->store_data_size = 0;
1331     if (buf && sz)
1332     {
1333         zh->store_data_buf = xmalloc(sz);
1334         zh->store_data_size = sz;
1335         memcpy(zh->store_data_buf, buf, sz);
1336     }
1337 }
1338
1339 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1340 {
1341     ZebraHandle zh = (ZebraHandle) p->handle;
1342     xfree(zh->store_data_buf);
1343     zh->store_data_buf = 0;
1344     zh->store_data_size = 0;
1345     p->setStoreData = extract_set_store_data_cb;
1346 }
1347
1348 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1349 {
1350     ZebraHandle zh = (ZebraHandle) p->handle;
1351     zebraExplain_addSchema (zh->reg->zei, oid);
1352 }
1353
1354 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1355                              int cmd, zebra_rec_keys_t reckeys)
1356 {
1357     if (zebra_rec_keys_rewind(reckeys))
1358     {
1359         SortIdx sortIdx = zh->reg->sortIdx;
1360         size_t slen;
1361         const char *str;
1362         struct it_key key_in;
1363
1364         sortIdx_sysno (sortIdx, sysno);
1365
1366         while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1367         {
1368             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1369
1370             sortIdx_type(sortIdx, ord);
1371             if (cmd == 1)
1372                 sortIdx_add(sortIdx, str, slen);
1373             else
1374                 sortIdx_add(sortIdx, "", 1);
1375         }
1376     }
1377 }
1378
1379 /*
1380  * Local variables:
1381  * c-basic-offset: 4
1382  * indent-tabs-mode: nil
1383  * End:
1384  * vim: shiftwidth=4 tabstop=8 expandtab
1385  */
1386