index/extract.c

   1 /* $Id: extract.c,v 1.241 2006-11-21 22:17:49 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #endif
  29 #if HAVE_UNISTD_H
  30 #include <unistd.h>
  31 #endif
  32 #include <fcntl.h>
  33
  34 #include "index.h"
  35 #include "orddict.h"
  36 #include <direntz.h>
  37 #include <charmap.h>
  38
  39 static int log_level_extract = 0;
  40 static int log_level_details = 0;
  41 static int log_level_initialized = 0;
  42
  43 static void zebra_init_log_level(void)
  44 {
  45     if (!log_level_initialized)
  46     {
  47         log_level_initialized = 1;
  48
  49         log_level_extract = yaz_log_module_level("extract");
  50         log_level_details = yaz_log_module_level("indexdetails");
  51     }
  52 }
  53
  54 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
  55                                       int cmd, zebra_rec_keys_t reckeys,
  56                                       zint staticrank);
  57 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
  58                                     int cmd, zebra_rec_keys_t skp);
  59 static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid);
  60 static void extract_token_add (RecWord *p);
  61
  62 static void logRecord (ZebraHandle zh)
  63 {
  64     ++zh->records_processed;
  65     if (!(zh->records_processed % 1000))
  66     {
  67         yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
  68                 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
  69                 zh->records_processed, zh->records_inserted,
  70                 zh->records_updated, zh->records_deleted);
  71     }
  72 }
  73
  74 static void extract_add_index_string (RecWord *p,
  75                                       zinfo_index_category_t cat,
  76                                       const char *str, int length);
  77
  78 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
  79
  80 static void extract_init(struct recExtractCtrl *p, RecWord *w)
  81 {
  82     w->seqno = 1;
  83     w->index_name = "any";
  84     w->index_type = 'w';
  85     w->extractCtrl = p;
  86     w->record_id = 0;
  87     w->section_id = 0;
  88     w->segment = 0;
  89 }
  90
  91 static void searchRecordKey(ZebraHandle zh,
  92                             zebra_rec_keys_t reckeys,
  93                             const char *index_name,
  94                             const char **ws, int ws_length)
  95 {
  96     int i;
  97     int ch = -1;
  98     zinfo_index_category_t cat = zinfo_index_category_index;
  99
 100     for (i = 0; i<ws_length; i++)
 101         ws[i] = NULL;
 102
 103     if (ch < 0)
 104         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, '0', index_name);
 105     if (ch < 0)
 106         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'p', index_name);
 107     if (ch < 0)
 108         ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, 'w', index_name);
 109
 110     if (ch < 0)
 111         return ;
 112
 113     if (zebra_rec_keys_rewind(reckeys))
 114     {
 115         zint startSeq = -1;
 116         const char *str;
 117         size_t slen;
 118         struct it_key key;
 119         zint seqno;
 120         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
 121         {
 122             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
 123
 124             seqno = key.mem[key.len-1];
 125
 126             if (key.mem[0] == ch)
 127             {
 128                 zint woff;
 129
 130                 if (startSeq == -1)
 131                     startSeq = seqno;
 132                 woff = seqno - startSeq;
 133                 if (woff >= 0 && woff < ws_length)
 134                     ws[woff] = str;
 135             }
 136         }
 137     }
 138 }
 139
 140 #define FILE_MATCH_BLANK "\t "
 141
 142 static char *get_match_from_spec(ZebraHandle zh,
 143                           zebra_rec_keys_t reckeys,
 144                           const char *fname, const char *spec)
 145 {
 146     static char dstBuf[2048];      /* static here ??? */
 147     char *dst = dstBuf;
 148     const char *s = spec;
 149
 150     while (1)
 151     {
 152         for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
 153             ;
 154         if (!*s)
 155             break;
 156         if (*s == '(')
 157         {
 158             const char *ws[32];
 159             char attset_str[64], attname_str[64];
 160             int i;
 161             int first = 1;
 162
 163             for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 164                 ;
 165             for (i = 0; *s && *s != ',' && *s != ')' &&
 166                      !strchr(FILE_MATCH_BLANK, *s); s++)
 167                 if (i+1 < sizeof(attset_str))
 168                     attset_str[i++] = *s;
 169             attset_str[i] = '\0';
 170
 171             for (; strchr(FILE_MATCH_BLANK, *s); s++)
 172                 ;
 173             if (*s != ',')
 174                 strcpy(attname_str, attset_str);
 175             else
 176             {
 177                 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
 178                     ;
 179                 for (i = 0; *s && *s != ')' &&
 180                          !strchr(FILE_MATCH_BLANK, *s); s++)
 181                     if (i+1 < sizeof(attname_str))
 182                         attname_str[i++] = *s;
 183                 attname_str[i] = '\0';
 184             }
 185
 186             searchRecordKey (zh, reckeys, attname_str, ws, 32);
 187
 188             if (*s != ')')
 189             {
 190                 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
 191                       spec, zh->m_group ? zh->m_group : "none");
 192                 return NULL;
 193             }
 194             s++;
 195
 196             for (i = 0; i<32; i++)
 197                 if (ws[i])
 198                 {
 199                     if (first)
 200                     {
 201                         *dst++ = ' ';
 202                         first = 0;
 203                     }
 204                     strcpy (dst, ws[i]);
 205                     dst += strlen(ws[i]);
 206                 }
 207             if (first)
 208             {
 209                 yaz_log (YLOG_WARN, "Record didn't contain match"
 210                       " fields in (%s,%s)", attset_str, attname_str);
 211                 return NULL;
 212             }
 213         }
 214         else if (*s == '$')
 215         {
 216             int spec_len;
 217             char special[64];
 218             const char *spec_src = NULL;
 219             const char *s1 = ++s;
 220             while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
 221                 s1++;
 222
 223             spec_len = s1 - s;
 224             if (spec_len > sizeof(special)-1)
 225                 spec_len = sizeof(special)-1;
 226             memcpy (special, s, spec_len);
 227             special[spec_len] = '\0';
 228             s = s1;
 229
 230             if (!strcmp (special, "group"))
 231                 spec_src = zh->m_group;
 232             else if (!strcmp (special, "database"))
 233                 spec_src = zh->basenames[0];
 234             else if (!strcmp (special, "filename")) {
 235                 spec_src = fname;
 236             }
 237             else if (!strcmp (special, "type"))
 238                 spec_src = zh->m_record_type;
 239             else
 240                 spec_src = NULL;
 241             if (spec_src)
 242             {
 243                 strcpy (dst, spec_src);
 244                 dst += strlen (spec_src);
 245             }
 246         }
 247         else if (*s == '\"' || *s == '\'')
 248         {
 249             int stopMarker = *s++;
 250             char tmpString[64];
 251             int i = 0;
 252
 253             while (*s && *s != stopMarker)
 254             {
 255                 if (i+1 < sizeof(tmpString))
 256                     tmpString[i++] = *s++;
 257             }
 258             if (*s)
 259                 s++;
 260             tmpString[i] = '\0';
 261             strcpy (dst, tmpString);
 262             dst += strlen (tmpString);
 263         }
 264         else
 265         {
 266             yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
 267                   spec, zh->m_group ? zh->m_group : "none");
 268             return NULL;
 269         }
 270         *dst++ = 1;
 271     }
 272     if (dst == dstBuf)
 273     {
 274         yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
 275               fname, zh->m_group ? zh->m_group : "none");
 276         return NULL;
 277     }
 278     *dst = '\0';
 279     return dstBuf;
 280 }
 281
 282 struct recordLogInfo {
 283     const char *fname;
 284     int recordOffset;
 285     struct recordGroup *rGroup;
 286 };
 287
 288 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
 289 {
 290     int i;
 291     for (i = 0; i<256; i++)
 292     {
 293         if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
 294             ctrl->seqno[i] = 1;
 295         else
 296             ctrl->seqno[i] = 0;
 297     }
 298     ctrl->flagShowRecords = !zh->m_flag_rw;
 299 }
 300
 301 static void all_matches_add(struct recExtractCtrl *ctrl)
 302 {
 303     RecWord word;
 304     extract_init(ctrl, &word);
 305     word.index_name = "_ALLRECORDS";
 306     word.index_type = 'w';
 307     word.seqno = 1;
 308     extract_add_index_string (&word, zinfo_index_category_alwaysmatches,
 309                               "", 0);
 310 }
 311
 312 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
 313                              int deleteFlag)
 314 {
 315     ZEBRA_RES r = ZEBRA_OK;
 316     int i, fd;
 317     char gprefix[128];
 318     char ext[128];
 319     char ext_res[128];
 320     struct file_read_info *fi = 0;
 321     const char *original_record_type = 0;
 322     RecType recType;
 323     void *recTypeClientData;
 324     struct ZebraRecStream stream, *streamp;
 325
 326     zebra_init_log_level();
 327
 328     if (!zh->m_group || !*zh->m_group)
 329         *gprefix = '\0';
 330     else
 331         sprintf (gprefix, "%s.", zh->m_group);
 332
 333     yaz_log(log_level_extract, "zebra_extract_file %s", fname);
 334
 335     /* determine file extension */
 336     *ext = '\0';
 337     for (i = strlen(fname); --i >= 0; )
 338         if (fname[i] == '/')
 339             break;
 340         else if (fname[i] == '.')
 341         {
 342             strcpy (ext, fname+i+1);
 343             break;
 344         }
 345     /* determine file type - depending on extension */
 346     original_record_type = zh->m_record_type;
 347     if (!zh->m_record_type)
 348     {
 349         sprintf (ext_res, "%srecordType.%s", gprefix, ext);
 350         zh->m_record_type = res_get (zh->res, ext_res);
 351     }
 352     if (!zh->m_record_type)
 353     {
 354         if (zh->records_processed < zh->m_file_verbose_limit)
 355             yaz_log (YLOG_LOG, "? %s", fname);
 356         return 0;
 357     }
 358     /* determine match criteria */
 359     if (!zh->m_record_id)
 360     {
 361         sprintf (ext_res, "%srecordId.%s", gprefix, ext);
 362         zh->m_record_id = res_get (zh->res, ext_res);
 363     }
 364
 365     if (!(recType =
 366           recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
 367                           &recTypeClientData)))
 368     {
 369         yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
 370         return ZEBRA_FAIL;
 371     }
 372
 373     switch(recType->version)
 374     {
 375     case 0:
 376         break;
 377     default:
 378         yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
 379     }
 380     if (sysno && deleteFlag)
 381     {
 382         streamp = 0;
 383         fi = 0;
 384     }
 385     else
 386     {
 387         char full_rep[1024];
 388
 389         if (zh->path_reg && !yaz_is_abspath (fname))
 390         {
 391             strcpy (full_rep, zh->path_reg);
 392             strcat (full_rep, "/");
 393             strcat (full_rep, fname);
 394         }
 395         else
 396             strcpy (full_rep, fname);
 397
 398         if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
 399         {
 400             yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
 401             zh->m_record_type = original_record_type;
 402             return ZEBRA_FAIL;
 403         }
 404         streamp = &stream;
 405         zebra_create_stream_fd(streamp, fd, 0);
 406     }
 407     while(1)
 408     {
 409         r = zebra_extract_record_stream(zh, streamp,
 410                                         deleteFlag,
 411                                         0, /* tst_mode */
 412                                         zh->m_record_type,
 413                                         sysno,
 414                                         0, /*match_criteria */
 415                                         fname,
 416                                         1, /* force_update */
 417                                         1, /* allow_update */
 418                                         recType, recTypeClientData);
 419         if (r != ZEBRA_OK)
 420         {
 421             break;
 422         }
 423         if (sysno)
 424         {
 425             break;
 426         }
 427     }
 428     if (streamp)
 429         stream.destroy(streamp);
 430     zh->m_record_type = original_record_type;
 431     return r;
 432 }
 433
 434 /*
 435   If sysno is provided, then it's used to identify the reocord.
 436   If not, and match_criteria is provided, then sysno is guessed
 437   If not, and a record is provided, then sysno is got from there
 438
 439  */
 440
 441 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
 442                                       const char *buf, size_t buf_size,
 443                                       int delete_flag,
 444                                       int test_mode,
 445                                       const char *recordType,
 446                                       zint *sysno,
 447                                       const char *match_criteria,
 448                                       const char *fname,
 449                                       int force_update,
 450                                       int allow_update)
 451 {
 452     struct ZebraRecStream stream;
 453     ZEBRA_RES res;
 454     void *clientData;
 455     RecType recType = 0;
 456
 457     if (recordType && *recordType)
 458     {
 459         yaz_log(log_level_extract,
 460                 "Record type explicitly specified: %s", recordType);
 461         recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
 462                                   &clientData);
 463     }
 464     else
 465     {
 466         if (!(zh->m_record_type))
 467         {
 468             yaz_log (YLOG_WARN, "No such record type defined");
 469             return ZEBRA_FAIL;
 470         }
 471         yaz_log(log_level_extract, "Get record type from rgroup: %s",
 472                 zh->m_record_type);
 473         recType = recType_byName (zh->reg->recTypes, zh->res,
 474                                   zh->m_record_type, &clientData);
 475         recordType = zh->m_record_type;
 476     }
 477
 478     if (!recType)
 479     {
 480         yaz_log (YLOG_WARN, "No such record type: %s", recordType);
 481         return ZEBRA_FAIL;
 482     }
 483
 484
 485
 486     zebra_create_stream_mem(&stream, buf, buf_size);
 487
 488     res = zebra_extract_record_stream(zh, &stream,
 489                                       delete_flag,
 490                                       test_mode,
 491                                       recordType,
 492                                       sysno,
 493                                       match_criteria,
 494                                       fname,
 495                                       force_update,
 496                                       allow_update,
 497                                       recType, clientData);
 498     stream.destroy(&stream);
 499     return res;
 500 }
 501
 502
 503 ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
 504                                       struct ZebraRecStream *stream,
 505                                       int delete_flag,
 506                                       int test_mode,
 507                                       const char *recordType,
 508                                       zint *sysno,
 509                                       const char *match_criteria,
 510                                       const char *fname,
 511                                       int force_update,
 512                                       int allow_update,
 513                                       RecType recType,
 514                                       void *recTypeClientData)
 515
 516 {
 517     zint sysno0 = 0;
 518     RecordAttr *recordAttr;
 519     struct recExtractCtrl extractCtrl;
 520     int r;
 521     const char *matchStr = 0;
 522     Record rec;
 523     off_t start_offset = 0;
 524     const char *pr_fname = fname;  /* filename to print .. */
 525     int show_progress = zh->records_processed < zh->m_file_verbose_limit ? 1:0;
 526
 527     zebra_init_log_level();
 528
 529     if (!pr_fname)
 530         pr_fname = "<no file>";  /* make it printable if file is omitted */
 531
 532     zebra_rec_keys_reset(zh->reg->keys);
 533     zebra_rec_keys_reset(zh->reg->sortKeys);
 534
 535     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
 536     {
 537         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
 538                                       zh->m_explain_database))
 539             return ZEBRA_FAIL;
 540     }
 541
 542     if (stream)
 543     {
 544         off_t null_offset = 0;
 545         extractCtrl.stream = stream;
 546
 547         start_offset = stream->tellf(stream);
 548
 549         extractCtrl.first_record = start_offset ? 0 : 1;
 550
 551         stream->endf(stream, &null_offset);;
 552
 553         extractCtrl.init = extract_init;
 554         extractCtrl.tokenAdd = extract_token_add;
 555         extractCtrl.schemaAdd = extract_schema_add;
 556         extractCtrl.dh = zh->reg->dh;
 557         extractCtrl.handle = zh;
 558         extractCtrl.match_criteria[0] = '\0';
 559         extractCtrl.staticrank = 0;
 560
 561
 562         init_extractCtrl(zh, &extractCtrl);
 563
 564         extract_set_store_data_prepare(&extractCtrl);
 565
 566         r = (*recType->extract)(recTypeClientData, &extractCtrl);
 567
 568         if (r == RECCTRL_EXTRACT_EOF)
 569             return ZEBRA_FAIL;
 570         else if (r == RECCTRL_EXTRACT_ERROR_GENERIC)
 571         {
 572             /* error occured during extraction ... */
 573             yaz_log (YLOG_WARN, "extract error: generic");
 574             return ZEBRA_FAIL;
 575         }
 576         else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER)
 577         {
 578             /* error occured during extraction ... */
 579             yaz_log (YLOG_WARN, "extract error: no such filter");
 580             return ZEBRA_FAIL;
 581         }
 582
 583         all_matches_add(&extractCtrl);
 584
 585         if (extractCtrl.match_criteria[0])
 586             match_criteria = extractCtrl.match_criteria;
 587     }
 588     if (!sysno) {
 589
 590         sysno = &sysno0;
 591
 592         if (match_criteria && *match_criteria) {
 593             matchStr = match_criteria;
 594         } else {
 595             if (zh->m_record_id && *zh->m_record_id) {
 596                 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
 597                                                zh->m_record_id);
 598                 if (!matchStr)
 599                 {
 600                     yaz_log(YLOG_WARN, "Bad match criteria (recordID)");
 601                     return ZEBRA_FAIL;
 602                 }
 603             }
 604         }
 605         if (matchStr)
 606         {
 607             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 608             char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
 609                                           matchStr);
 610             if (rinfo)
 611             {
 612                 assert(*rinfo == sizeof(*sysno));
 613                 memcpy (sysno, rinfo+1, sizeof(*sysno));
 614             }
 615         }
 616     }
 617     if (zebra_rec_keys_empty(zh->reg->keys))
 618     {
 619         /* the extraction process returned no information - the record
 620            is probably empty - unless flagShowRecords is in use */
 621         if (test_mode)
 622             return ZEBRA_OK;
 623     }
 624
 625     if (! *sysno)
 626     {
 627         /* new record */
 628         if (delete_flag)
 629         {
 630             yaz_log (YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 631                          pr_fname, (zint) start_offset);
 632             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
 633             return ZEBRA_FAIL;
 634         }
 635         if (show_progress)
 636             yaz_log (YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
 637                      (zint) start_offset);
 638         rec = rec_new (zh->reg->records);
 639
 640         *sysno = rec->sysno;
 641
 642         recordAttr = rec_init_attr (zh->reg->zei, rec);
 643         recordAttr->staticrank = extractCtrl.staticrank;
 644
 645         if (matchStr)
 646         {
 647             int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 648             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
 649                             sizeof(*sysno), sysno);
 650         }
 651
 652         extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 653         extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 654                                   recordAttr->staticrank);
 655         zh->records_inserted++;
 656     }
 657     else
 658     {
 659         /* record already exists */
 660         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 661         zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
 662         if (!allow_update)
 663         {
 664             yaz_log (YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
 665                          recordType, pr_fname, (zint) start_offset);
 666             logRecord(zh);
 667             return ZEBRA_FAIL;
 668         }
 669
 670         rec = rec_get (zh->reg->records, *sysno);
 671         assert (rec);
 672
 673         recordAttr = rec_init_attr (zh->reg->zei, rec);
 674
 675         zebra_rec_keys_set_buf(delkeys,
 676                                rec->info[recInfo_delKeys],
 677                                rec->size[recInfo_delKeys],
 678                                0);
 679         zebra_rec_keys_set_buf(sortKeys,
 680                                rec->info[recInfo_sortKeys],
 681                                rec->size[recInfo_sortKeys],
 682                                0);
 683
 684         extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
 685         extract_flush_record_keys(zh, *sysno, 0, delkeys,
 686                                   recordAttr->staticrank);
 687         if (delete_flag)
 688         {
 689             /* record going to be deleted */
 690             if (zebra_rec_keys_empty(delkeys))
 691             {
 692                 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 693                         pr_fname, (zint) start_offset);
 694                 yaz_log(YLOG_WARN, "cannot delete file above, "
 695                         "storeKeys false (3)");
 696             }
 697             else
 698             {
 699                 if (show_progress)
 700                     yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
 701                             pr_fname, (zint) start_offset);
 702                 zh->records_deleted++;
 703                 if (matchStr)
 704                 {
 705                     int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
 706                     dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
 707                 }
 708                 rec_del (zh->reg->records, &rec);
 709             }
 710             rec_free(&rec);
 711             logRecord(zh);
 712             return ZEBRA_OK;
 713         }
 714         else
 715         {
 716             if (show_progress)
 717                     yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
 718                             pr_fname, (zint) ZINT_FORMAT);
 719             recordAttr->staticrank = extractCtrl.staticrank;
 720             extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
 721             extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
 722                                       recordAttr->staticrank);
 723             zh->records_updated++;
 724         }
 725         zebra_rec_keys_close(delkeys);
 726         zebra_rec_keys_close(sortKeys);
 727     }
 728     /* update file type */
 729     xfree (rec->info[recInfo_fileType]);
 730     rec->info[recInfo_fileType] =
 731         rec_strdup (recordType, &rec->size[recInfo_fileType]);
 732
 733     /* update filename */
 734     xfree (rec->info[recInfo_filename]);
 735     rec->info[recInfo_filename] =
 736         rec_strdup (fname, &rec->size[recInfo_filename]);
 737
 738     /* update delete keys */
 739     xfree (rec->info[recInfo_delKeys]);
 740     if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
 741     {
 742         zebra_rec_keys_get_buf(zh->reg->keys,
 743                                &rec->info[recInfo_delKeys],
 744                                &rec->size[recInfo_delKeys]);
 745     }
 746     else
 747     {
 748         rec->info[recInfo_delKeys] = NULL;
 749         rec->size[recInfo_delKeys] = 0;
 750     }
 751     /* update sort keys */
 752     xfree (rec->info[recInfo_sortKeys]);
 753
 754     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 755                            &rec->info[recInfo_sortKeys],
 756                            &rec->size[recInfo_sortKeys]);
 757
 758     /* save file size of original record */
 759     zebraExplain_recordBytesIncrement (zh->reg->zei,
 760                                        - recordAttr->recordSize);
 761     if (stream)
 762     {
 763         off_t end_offset = stream->endf(stream, 0);
 764
 765         if (!end_offset)
 766             end_offset = stream->tellf(stream);
 767         else
 768             stream->seekf(stream, end_offset);
 769
 770         recordAttr->recordSize = end_offset - start_offset;
 771         zebraExplain_recordBytesIncrement(zh->reg->zei,
 772                                           recordAttr->recordSize);
 773     }
 774
 775     /* set run-number for this record */
 776     recordAttr->runNumber =
 777         zebraExplain_runNumberIncrement (zh->reg->zei, 0);
 778
 779     /* update store data */
 780     xfree (rec->info[recInfo_storeData]);
 781
 782     /* update store data */
 783     if (zh->store_data_buf)
 784     {
 785         rec->size[recInfo_storeData] = zh->store_data_size;
 786         rec->info[recInfo_storeData] = zh->store_data_buf;
 787         zh->store_data_buf = 0;
 788         recordAttr->recordSize = zh->store_data_size;
 789     }
 790     else if (zh->m_store_data)
 791     {
 792         off_t cur_offset = stream->tellf(stream);
 793
 794         rec->size[recInfo_storeData] = recordAttr->recordSize;
 795         rec->info[recInfo_storeData] = (char *)
 796             xmalloc (recordAttr->recordSize);
 797         stream->seekf(stream, start_offset);
 798         stream->readf(stream, rec->info[recInfo_storeData],
 799                       recordAttr->recordSize);
 800         stream->seekf(stream, cur_offset);
 801     }
 802     else
 803     {
 804         rec->info[recInfo_storeData] = NULL;
 805         rec->size[recInfo_storeData] = 0;
 806     }
 807     /* update database name */
 808     xfree (rec->info[recInfo_databaseName]);
 809     rec->info[recInfo_databaseName] =
 810         rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
 811
 812     /* update offset */
 813     recordAttr->recordOffset = start_offset;
 814
 815     /* commit this record */
 816     rec_put (zh->reg->records, &rec);
 817     logRecord(zh);
 818     return ZEBRA_OK;
 819 }
 820
 821 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
 822 {
 823     ZebraHandle zh = (ZebraHandle) handle;
 824     struct recExtractCtrl extractCtrl;
 825
 826     if (zebraExplain_curDatabase (zh->reg->zei,
 827                                   rec->info[recInfo_databaseName]))
 828     {
 829         abort();
 830         if (zebraExplain_newDatabase (zh->reg->zei,
 831                                       rec->info[recInfo_databaseName], 0))
 832             abort ();
 833     }
 834
 835     zebra_rec_keys_reset(zh->reg->keys);
 836     zebra_rec_keys_reset(zh->reg->sortKeys);
 837
 838     extractCtrl.init = extract_init;
 839     extractCtrl.tokenAdd = extract_token_add;
 840     extractCtrl.schemaAdd = extract_schema_add;
 841     extractCtrl.dh = zh->reg->dh;
 842
 843     init_extractCtrl(zh, &extractCtrl);
 844
 845     extractCtrl.flagShowRecords = 0;
 846     extractCtrl.match_criteria[0] = '\0';
 847     extractCtrl.staticrank = 0;
 848     extractCtrl.handle = handle;
 849     extractCtrl.first_record = 1;
 850
 851     extract_set_store_data_prepare(&extractCtrl);
 852
 853     if (n)
 854         grs_extract_tree(&extractCtrl, n);
 855
 856     if (rec->size[recInfo_delKeys])
 857     {
 858         zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 859
 860         zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
 861
 862         zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
 863                                rec->size[recInfo_delKeys],
 864                                0);
 865         extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
 866         zebra_rec_keys_close(delkeys);
 867
 868         zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
 869                                rec->size[recInfo_sortKeys],
 870                                0);
 871
 872         extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
 873         zebra_rec_keys_close(sortkeys);
 874     }
 875     extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
 876     extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
 877
 878     xfree (rec->info[recInfo_delKeys]);
 879     zebra_rec_keys_get_buf(zh->reg->keys,
 880                            &rec->info[recInfo_delKeys],
 881                            &rec->size[recInfo_delKeys]);
 882
 883     xfree (rec->info[recInfo_sortKeys]);
 884     zebra_rec_keys_get_buf(zh->reg->sortKeys,
 885                            &rec->info[recInfo_sortKeys],
 886                            &rec->size[recInfo_sortKeys]);
 887     return ZEBRA_OK;
 888 }
 889
 890 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
 891                           zebra_rec_keys_t reckeys,
 892                           int level)
 893 {
 894     if (zebra_rec_keys_rewind(reckeys))
 895     {
 896         size_t slen;
 897         const char *str;
 898         struct it_key key;
 899         NMEM nmem = nmem_create();
 900
 901         while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
 902         {
 903             char keystr[200]; /* room for zints to print */
 904             char *dst_term = 0;
 905             int ord = CAST_ZINT_TO_INT(key.mem[0]);
 906             int index_type, i;
 907             const char *string_index;
 908
 909             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
 910                                     0/* db */, &string_index);
 911             assert(index_type);
 912             zebra_term_untrans_iconv(zh, nmem, index_type,
 913                                      &dst_term, str);
 914             *keystr = '\0';
 915             for (i = 0; i<key.len; i++)
 916             {
 917                 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key.mem[i]);
 918             }
 919
 920             if (*str < CHR_BASE_CHAR)
 921             {
 922                 int i;
 923                 char dst_buf[200]; /* room for special chars */
 924
 925                 strcpy(dst_buf , "?");
 926
 927                 if (!strcmp(str, ""))
 928                     strcpy(dst_buf, "alwaysmatches");
 929                 if (!strcmp(str, FIRST_IN_FIELD_STR))
 930                     strcpy(dst_buf, "firstinfield");
 931                 else if (!strcmp(str, CHR_UNKNOWN))
 932                     strcpy(dst_buf, "unknown");
 933                 else if (!strcmp(str, CHR_SPACE))
 934                     strcpy(dst_buf, "space");
 935
 936                 for (i = 0; i<slen; i++)
 937                 {
 938                     sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
 939                 }
 940                 yaz_log(level, "%s%c %s %s", keystr, index_type,
 941                         string_index, dst_buf);
 942
 943             }
 944             else
 945                 yaz_log(level, "%s%c %s \"%s\"", keystr, index_type,
 946                         string_index, dst_term);
 947
 948             nmem_reset(nmem);
 949         }
 950         nmem_destroy(nmem);
 951     }
 952 }
 953
 954 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
 955                              zebra_rec_keys_t reckeys)
 956 {
 957     ZebraExplainInfo zei = zh->reg->zei;
 958     struct ord_stat {
 959         int no;
 960         int ord;
 961         struct ord_stat *next;
 962     };
 963
 964     if (zebra_rec_keys_rewind(reckeys))
 965     {
 966         struct ord_stat *ord_list = 0;
 967         struct ord_stat *p;
 968         size_t slen;
 969         const char *str;
 970         struct it_key key_in;
 971         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
 972         {
 973             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
 974
 975             for (p = ord_list; p ; p = p->next)
 976                 if (p->ord == ord)
 977                 {
 978                     p->no++;
 979                     break;
 980                 }
 981             if (!p)
 982             {
 983                 p = xmalloc(sizeof(*p));
 984                 p->no = 1;
 985                 p->ord = ord;
 986                 p->next = ord_list;
 987                 ord_list = p;
 988             }
 989         }
 990
 991         p = ord_list;
 992         while (p)
 993         {
 994             struct ord_stat *p1 = p;
 995
 996             if (is_insert)
 997                 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
 998             else
 999                 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1000             p = p->next;
1001             xfree(p1);
1002         }
1003     }
1004 }
1005
1006 void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd,
1007                                zebra_rec_keys_t reckeys,
1008                                zint staticrank)
1009 {
1010     ZebraExplainInfo zei = zh->reg->zei;
1011
1012     extract_rec_keys_adjust(zh, cmd, reckeys);
1013
1014     if (log_level_details)
1015     {
1016         yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1017                 sysno, cmd ? "insert" : "delete");
1018         extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1019     }
1020
1021     if (!zh->reg->key_block)
1022     {
1023         int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1024         const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", ".");
1025         int use_threads = atoi(res_get_def (zh->res, "threads", "1"));
1026         zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1027     }
1028     zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1029
1030     if (zebra_rec_keys_rewind(reckeys))
1031     {
1032         size_t slen;
1033         const char *str;
1034         struct it_key key_in;
1035         while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1036         {
1037             key_block_write(zh->reg->key_block, sysno,
1038                             &key_in, cmd, str, slen,
1039                             staticrank, zh->m_staticrank);
1040         }
1041     }
1042 }
1043
1044
1045 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
1046                                      zebra_rec_keys_t reckeys,
1047                                      zebra_snippets *snippets)
1048 {
1049     NMEM nmem = nmem_create();
1050     if (zebra_rec_keys_rewind(reckeys))
1051     {
1052         const char *str;
1053         size_t slen;
1054         struct it_key key;
1055         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1056         {
1057             char dst_buf[IT_MAX_WORD];
1058             char *dst_term = dst_buf;
1059             int ord;
1060             zint seqno;
1061             int index_type;
1062
1063             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1064             seqno = key.mem[key.len-1];
1065             ord = CAST_ZINT_TO_INT(key.mem[0]);
1066
1067             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1068                                     0/* db */, 0 /* string_index */);
1069             assert(index_type);
1070             zebra_term_untrans_iconv(zh, nmem, index_type,
1071                                      &dst_term, str);
1072             zebra_snippets_append(snippets, seqno, ord, dst_term);
1073             nmem_reset(nmem);
1074         }
1075     }
1076     nmem_destroy(nmem);
1077     return ZEBRA_OK;
1078 }
1079
1080 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1081 {
1082     yaz_log(YLOG_LOG, "print_rec_keys");
1083     if (zebra_rec_keys_rewind(reckeys))
1084     {
1085         const char *str;
1086         size_t slen;
1087         struct it_key key;
1088         while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1089         {
1090             char dst_buf[IT_MAX_WORD];
1091             zint seqno;
1092             int index_type;
1093             int ord = CAST_ZINT_TO_INT(key.mem[0]);
1094             const char *db = 0;
1095             assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1096
1097             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1098
1099             seqno = key.mem[key.len-1];
1100
1101             zebra_term_untrans(zh, index_type, dst_buf, str);
1102
1103             yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1104                     " term=%s", ord, seqno, dst_buf);
1105         }
1106     }
1107 }
1108
1109 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1110                                      const char *str, int length)
1111 {
1112     struct it_key key;
1113     ZebraHandle zh = p->extractCtrl->handle;
1114     ZebraExplainInfo zei = zh->reg->zei;
1115     int ch, i;
1116
1117     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1118     if (ch < 0)
1119         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1120
1121     i = 0;
1122     key.mem[i++] = ch;
1123     key.mem[i++] = p->record_id;
1124     key.mem[i++] = p->section_id;
1125
1126     if (zh->m_segment_indexing)
1127         key.mem[i++] = p->segment;
1128     key.mem[i++] = p->seqno;
1129     key.len = i;
1130
1131     zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1132 }
1133
1134 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1135 {
1136     struct it_key key;
1137     ZebraHandle zh = p->extractCtrl->handle;
1138     ZebraExplainInfo zei = zh->reg->zei;
1139     int ch;
1140     zinfo_index_category_t cat = zinfo_index_category_sort;
1141
1142     ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1143     if (ch < 0)
1144         ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1145     key.len = 2;
1146     key.mem[0] = ch;
1147     key.mem[1] = p->record_id;
1148
1149     zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1150 }
1151
1152 static void extract_add_string(RecWord *p, const char *string, int length)
1153 {
1154     ZebraHandle zh = p->extractCtrl->handle;
1155     assert (length > 0);
1156
1157     if (!p->index_name)
1158         return;
1159
1160     if (zebra_maps_is_sort(zh->reg->zebra_maps, p->index_type))
1161         extract_add_sort_string(p, string, length);
1162     else
1163     {
1164         extract_add_index_string(p, zinfo_index_category_index,
1165                                  string, length);
1166         if (zebra_maps_is_alwaysmatches(zh->reg->zebra_maps, p->index_type))
1167         {
1168             RecWord word;
1169             memcpy(&word, p, sizeof(word));
1170
1171             word.seqno = 1;
1172             extract_add_index_string(
1173                 &word, zinfo_index_category_alwaysmatches, "", 0);
1174         }
1175     }
1176 }
1177
1178 static void extract_add_incomplete_field(RecWord *p)
1179 {
1180     ZebraHandle zh = p->extractCtrl->handle;
1181     const char *b = p->term_buf;
1182     int remain = p->term_len;
1183     const char **map = 0;
1184
1185     if (remain > 0)
1186         map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1187
1188     if (map)
1189     {
1190         if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type))
1191         {
1192              /* first in field marker */
1193             extract_add_string(p, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1194             p->seqno++;
1195         }
1196     }
1197     while (map)
1198     {
1199         char buf[IT_MAX_WORD+1];
1200         int i, remain;
1201
1202         /* Skip spaces */
1203         while (map && *map && **map == *CHR_SPACE)
1204         {
1205             remain = p->term_len - (b - p->term_buf);
1206             if (remain > 0)
1207                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b,
1208                                        remain, 0);
1209             else
1210                 map = 0;
1211         }
1212         if (!map)
1213             break;
1214         i = 0;
1215         while (map && *map && **map != *CHR_SPACE)
1216         {
1217             const char *cp = *map;
1218
1219             while (i < IT_MAX_WORD && *cp)
1220                 buf[i++] = *(cp++);
1221             remain = p->term_len - (b - p->term_buf);
1222             if (remain > 0)
1223                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0);
1224             else
1225                 map = 0;
1226         }
1227         if (!i)
1228             return;
1229         extract_add_string (p, buf, i);
1230         p->seqno++;
1231     }
1232 }
1233
1234 static void extract_add_complete_field (RecWord *p)
1235 {
1236     ZebraHandle zh = p->extractCtrl->handle;
1237     const char *b = p->term_buf;
1238     char buf[IT_MAX_WORD+1];
1239     const char **map = 0;
1240     int i = 0, remain = p->term_len;
1241
1242     if (remain > 0)
1243         map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, remain, 1);
1244
1245     while (remain > 0 && i < IT_MAX_WORD)
1246     {
1247         while (map && *map && **map == *CHR_SPACE)
1248         {
1249             remain = p->term_len - (b - p->term_buf);
1250
1251             if (remain > 0)
1252             {
1253                 int first = i ? 0 : 1;  /* first position */
1254                 map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, first);
1255             }
1256             else
1257                 map = 0;
1258         }
1259         if (!map)
1260             break;
1261
1262         if (i && i < IT_MAX_WORD)
1263             buf[i++] = *CHR_SPACE;
1264         while (map && *map && **map != *CHR_SPACE)
1265         {
1266             const char *cp = *map;
1267
1268             if (**map == *CHR_CUT)
1269             {
1270                 i = 0;
1271             }
1272             else
1273             {
1274                 if (i >= IT_MAX_WORD)
1275                     break;
1276                 while (i < IT_MAX_WORD && *cp)
1277                     buf[i++] = *(cp++);
1278             }
1279             remain = p->term_len  - (b - p->term_buf);
1280             if (remain > 0)
1281             {
1282                 map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b,
1283                                         remain, 0);
1284             }
1285             else
1286                 map = 0;
1287         }
1288     }
1289     if (!i)
1290         return;
1291     extract_add_string (p, buf, i);
1292 }
1293
1294 static void extract_token_add(RecWord *p)
1295 {
1296     ZebraHandle zh = p->extractCtrl->handle;
1297     WRBUF wrbuf;
1298
1299     if (log_level_extract)
1300     {
1301         yaz_log(log_level_extract, "extract_token_add "
1302                 "type=%c index=%s seqno=" ZINT_FORMAT " s=%.*s",
1303                 p->index_type, p->index_name,
1304                 p->seqno, p->term_len, p->term_buf);
1305     }
1306     if ((wrbuf = zebra_replace(zh->reg->zebra_maps, p->index_type, 0,
1307                                p->term_buf, p->term_len)))
1308     {
1309         p->term_buf = wrbuf_buf(wrbuf);
1310         p->term_len = wrbuf_len(wrbuf);
1311     }
1312     if (zebra_maps_is_complete (zh->reg->zebra_maps, p->index_type))
1313         extract_add_complete_field (p);
1314     else
1315         extract_add_incomplete_field(p);
1316 }
1317
1318 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1319                                       void *buf, size_t sz)
1320 {
1321     ZebraHandle zh = (ZebraHandle) p->handle;
1322
1323     xfree(zh->store_data_buf);
1324     zh->store_data_buf = 0;
1325     zh->store_data_size = 0;
1326     if (buf && sz)
1327     {
1328         zh->store_data_buf = xmalloc(sz);
1329         zh->store_data_size = sz;
1330         memcpy(zh->store_data_buf, buf, sz);
1331     }
1332 }
1333
1334 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1335 {
1336     ZebraHandle zh = (ZebraHandle) p->handle;
1337     xfree(zh->store_data_buf);
1338     zh->store_data_buf = 0;
1339     zh->store_data_size = 0;
1340     p->setStoreData = extract_set_store_data_cb;
1341 }
1342
1343 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1344 {
1345     ZebraHandle zh = (ZebraHandle) p->handle;
1346     zebraExplain_addSchema (zh->reg->zei, oid);
1347 }
1348
1349 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1350                              int cmd, zebra_rec_keys_t reckeys)
1351 {
1352     if (zebra_rec_keys_rewind(reckeys))
1353     {
1354         SortIdx sortIdx = zh->reg->sortIdx;
1355         size_t slen;
1356         const char *str;
1357         struct it_key key_in;
1358
1359         sortIdx_sysno (sortIdx, sysno);
1360
1361         while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1362         {
1363             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1364
1365             sortIdx_type(sortIdx, ord);
1366             if (cmd == 1)
1367                 sortIdx_add(sortIdx, str, slen);
1368             else
1369                 sortIdx_add(sortIdx, "", 1);
1370         }
1371     }
1372 }
1373
1374 /*
1375  * Local variables:
1376  * c-basic-offset: 4
1377  * indent-tabs-mode: nil
1378  * End:
1379  * vim: shiftwidth=4 tabstop=8 expandtab
1380  */
1381