index/main.c

   1 /*
   2  * Copyright (C) 1994-2001, Index Data
   3  * All rights reserved.
   4  *
   5  * $Id: main.c,v 1.80 2001-11-19 23:05:22 adam Exp $
   6  */
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <assert.h>
  10 #ifdef WIN32
  11 #include <io.h>
  12 #else
  13 #include <unistd.h>
  14 #endif
  15
  16 #include <yaz/data1.h>
  17 #include "index.h"
  18 #include "recindex.h"
  19
  20 #ifndef ZEBRASDR
  21 #define ZEBRASDR 0
  22 #endif
  23
  24 #if ZEBRASDR
  25 #include "zebrasdr.h"
  26 #endif
  27
  28 char *prog;
  29
  30 Res common_resource = 0;
  31
  32
  33 int main (int argc, char **argv)
  34 {
  35     int ret;
  36     int cmd = 0;
  37     char *arg;
  38     char *configName = FNAME_CONFIG;
  39     int nsections = 0;
  40     int disableCommit = 0;
  41     size_t mem_max = 0;
  42     char nbuf[100];
  43     struct recordGroup rGroupDef;
  44
  45     nmem_init ();
  46
  47 #ifdef WIN32
  48 #else
  49     sprintf(nbuf, "%.40s(%d)", *argv, getpid());
  50     yaz_log_init_prefix (nbuf);
  51 #endif
  52
  53 #if ZEBRASDR
  54     zebraSdr_std ();
  55     rGroupDef.useSDR = 0;
  56 #endif
  57     rGroupDef.groupName = NULL;
  58     rGroupDef.databaseName = NULL;
  59     rGroupDef.path = NULL;
  60     rGroupDef.recordId = NULL;
  61     rGroupDef.recordType = NULL;
  62     rGroupDef.flagStoreData = -1;
  63     rGroupDef.flagStoreKeys = -1;
  64     rGroupDef.flagRw = 1;
  65     rGroupDef.databaseNamePath = 0;
  66     rGroupDef.explainDatabase = 0;
  67     rGroupDef.fileVerboseLimit = 100000;
  68     rGroupDef.zebra_maps = NULL;
  69     rGroupDef.dh = data1_create ();
  70     rGroupDef.recTypes = recTypes_init (rGroupDef.dh);
  71     recTypes_default_handlers (rGroupDef.recTypes);
  72
  73     prog = *argv;
  74     if (argc < 2)
  75     {
  76         fprintf (stderr, "%s [options] command <dir> ...\n"
  77         "Commands:\n"
  78         " update <dir>  Update index with files below <dir>.\n"
  79         "               If <dir> is empty filenames are read from stdin.\n"
  80         " delete <dir>  Delete index with files below <dir>.\n"
  81         " commit        Commit changes\n"
  82         " clean         Clean shadow files\n"
  83         "Options:\n"
  84         " -t <type>     Index files as <type> (grs or text).\n"
  85         " -c <config>   Read configuration file <config>.\n"
  86         " -g <group>    Index files according to group settings.\n"
  87         " -d <database> Records belong to Z39.50 database <database>.\n"
  88         " -m <mbytes>   Use <mbytes> before flushing keys to disk.\n"
  89         " -n            Don't use shadow system.\n"
  90         " -s            Show analysis on stdout, but do no work.\n"
  91         " -v <level>    Set logging to <level>.\n"
  92         " -l <file>     Write log to <file>.\n"
  93         " -f <n>        Display information for the first <n> records.\n"
  94 #if ZEBRASDR
  95         " -S            Use SDRKit\n"
  96 #endif
  97         " -V            Show version.\n", *argv
  98                  );
  99         exit (1);
 100     }
 101     while ((ret = options ("sVt:c:g:d:m:v:nf:l:"
 102 #if ZEBRASDR
 103                            "S"
 104 #endif
 105                            , argv, argc, &arg)) != -2)
 106     {
 107         if (ret == 0)
 108         {
 109             const char *rval;
 110             if(cmd == 0) /* command */
 111             {
 112                 if (!common_resource)
 113                 {
 114 #if ZMBOL
 115                     logf (LOG_LOG, "zmbol version %s %s",
 116                           ZEBRAVER, ZEBRADATE);
 117 #else
 118                     logf (LOG_LOG, "zebra version %s %s",
 119                           ZEBRAVER, ZEBRADATE);
 120 #endif
 121                     common_resource = res_open (configName ?
 122                                                 configName : FNAME_CONFIG);
 123                     if (!common_resource)
 124                     {
 125                         logf (LOG_FATAL, "cannot read file `%s'", configName);
 126                         exit (1);
 127                     }
 128                     data1_set_tabpath (rGroupDef.dh, res_get (common_resource,
 129                                                               "profilePath"));
 130
 131                     rGroupDef.bfs =
 132                         bfs_create (res_get (common_resource, "register"));
 133                     if (!rGroupDef.bfs)
 134                     {
 135                         logf (LOG_FATAL, "Cannot access register");
 136                         exit(1);
 137                     }
 138
 139                     bf_lockDir (rGroupDef.bfs,
 140                                 res_get (common_resource, "lockDir"));
 141                     rGroupDef.zebra_maps = zebra_maps_open (common_resource);
 142                 }
 143                 if (!strcmp (arg, "update"))
 144                     cmd = 'u';
 145                 else if (!strcmp (arg, "update1"))
 146                     cmd = 'U';
 147                 else if (!strcmp (arg, "update2"))
 148                     cmd = 'm';
 149                 else if (!strcmp (arg, "dump"))
 150                     cmd = 's';
 151                 else if (!strcmp (arg, "del") || !strcmp(arg, "delete"))
 152                     cmd = 'd';
 153                 else if (!strcmp (arg, "init"))
 154                 {
 155                     zebraIndexUnlock();
 156                     rval = res_get (common_resource, "shadow");
 157                     zebraIndexLock (rGroupDef.bfs, 0, rval);
 158                     if (rval && *rval)
 159                         bf_cache (rGroupDef.bfs, rval);
 160                     zebraIndexLockMsg ("w");
 161                     bf_reset (rGroupDef.bfs);
 162                 }
 163                 else if (!strcmp (arg, "commit"))
 164                 {
 165                     rval = res_get (common_resource, "shadow");
 166                     zebraIndexLock (rGroupDef.bfs, 1, rval);
 167                     if (rval && *rval)
 168                         bf_cache (rGroupDef.bfs, rval);
 169                     else
 170                     {
 171                         logf (LOG_FATAL, "Cannot perform commit");
 172                         logf (LOG_FATAL, "No shadow area defined");
 173                         exit (1);
 174                     }
 175                     if (bf_commitExists (rGroupDef.bfs))
 176                     {
 177                         logf (LOG_LOG, "commit start");
 178                         zebraIndexLockMsg ("c");
 179                         zebraIndexWait (1);
 180                         logf (LOG_LOG, "commit execute");
 181                         bf_commitExec (rGroupDef.bfs);
 182 #ifndef WIN32
 183                         sync ();
 184 #endif
 185                         zebraIndexLockMsg ("d");
 186                         zebraIndexWait (0);
 187                         logf (LOG_LOG, "commit clean");
 188                         bf_commitClean (rGroupDef.bfs, rval);
 189                     }
 190                     else
 191                         logf (LOG_LOG, "nothing to commit");
 192                 }
 193                 else if (!strcmp (arg, "clean"))
 194                 {
 195                     rval = res_get (common_resource, "shadow");
 196                     zebraIndexLock (rGroupDef.bfs, 1, rval);
 197                     if (bf_commitExists (rGroupDef.bfs))
 198                     {
 199                         zebraIndexLockMsg ("d");
 200                         zebraIndexWait (0);
 201                         logf (LOG_LOG, "commit clean");
 202                         bf_commitClean (rGroupDef.bfs, rval);
 203                     }
 204                     else
 205                         logf (LOG_LOG, "nothing to clean");
 206                 }
 207                 else if (!strcmp (arg, "stat") || !strcmp (arg, "status"))
 208                 {
 209                     Records records;
 210                     rval = res_get (common_resource, "shadow");
 211                     zebraIndexLock (rGroupDef.bfs, 0, rval);
 212                     if (rval && *rval)
 213                     {
 214                         bf_cache (rGroupDef.bfs, rval);
 215                         zebraIndexLockMsg ("r");
 216                     }
 217                     records = rec_open (rGroupDef.bfs, 0, 0);
 218                     rec_prstat (records);
 219                     rec_close (&records);
 220                     inv_prstat (rGroupDef.bfs);
 221                 }
 222                 else if (!strcmp (arg, "compact"))
 223                 {
 224                     rval = res_get (common_resource, "shadow");
 225                     zebraIndexLock (rGroupDef.bfs, 0, rval);
 226                     if (rval && *rval)
 227                     {
 228                         bf_cache (rGroupDef.bfs, rval);
 229                         zebraIndexLockMsg ("r");
 230                     }
 231                     inv_compact(rGroupDef.bfs);
 232                 }
 233                 else
 234                 {
 235                     logf (LOG_FATAL, "unknown command: %s", arg);
 236                     exit (1);
 237                 }
 238             }
 239             else
 240             {
 241                 struct recordGroup rGroup;
 242 #if ZMBOL
 243 #else
 244                 /* For zebra, delete lock file and reset register */
 245                 if (rGroupDef.flagRw)
 246                 {
 247                     zebraIndexUnlock();
 248                     bf_reset (rGroupDef.bfs);
 249                 }
 250 #endif
 251                 rval = res_get (common_resource, "shadow");
 252                 zebraIndexLock (rGroupDef.bfs, 0, rval);
 253                 if (rGroupDef.flagRw)
 254                 {
 255                     if (rval && *rval && !disableCommit)
 256                     {
 257                         bf_cache (rGroupDef.bfs, rval);
 258                         zebraIndexLockMsg ("r");
 259                     }
 260                     else
 261                     {
 262                         bf_cache (rGroupDef.bfs, 0);
 263                         zebraIndexLockMsg ("w");
 264                     }
 265                     zebraIndexWait (0);
 266                 }
 267                 memcpy (&rGroup, &rGroupDef, sizeof(rGroup));
 268                 rGroup.path = arg;
 269                 switch (cmd)
 270                 {
 271                 case 'u':
 272                     if (!key_open (&rGroup, mem_max))
 273                     {
 274                         logf (LOG_LOG, "updating %s", rGroup.path);
 275                         repositoryUpdate (&rGroup);
 276                         nsections = key_close (&rGroup);
 277                     }
 278                     break;
 279                 case 'U':
 280                     if (!key_open (&rGroup, mem_max))
 281                     {
 282                         logf (LOG_LOG, "updating (pass 1) %s", rGroup.path);
 283                         repositoryUpdate (&rGroup);
 284                         key_close (&rGroup);
 285                     }
 286                     nsections = 0;
 287                     break;
 288                 case 'd':
 289                     if (!key_open (&rGroup,mem_max))
 290                     {
 291                         logf (LOG_LOG, "deleting %s", rGroup.path);
 292                         repositoryDelete (&rGroup);
 293                         nsections = key_close (&rGroup);
 294                     }
 295                     break;
 296                 case 's':
 297                     logf (LOG_LOG, "dumping %s", rGroup.path);
 298                     repositoryShow (&rGroup);
 299                     nsections = 0;
 300                     break;
 301                 case 'm':
 302                     nsections = -1;
 303                     break;
 304                 default:
 305                     nsections = 0;
 306                 }
 307                 cmd = 0;
 308                 if (nsections)
 309                 {
 310                     logf (LOG_LOG, "merging with index");
 311                     key_input (rGroup.bfs, nsections, 60, common_resource);
 312 #ifndef WIN32
 313                     sync ();
 314 #endif
 315                 }
 316                 log_event_end (NULL, NULL);
 317             }
 318         }
 319         else if (ret == 'V')
 320         {
 321 #if ZMBOL
 322             fprintf (stderr, "Z'mbol %s %s\n", ZEBRAVER, ZEBRADATE);
 323 #else
 324             fprintf (stderr, "Zebra %s %s\n", ZEBRAVER, ZEBRADATE);
 325 #endif
 326             fprintf (stderr, " (C) 1994-2001, Index Data ApS\n");
 327 #ifdef WIN32
 328 #ifdef _DEBUG
 329             fprintf (stderr, " WIN32 Debug\n");
 330 #else
 331             fprintf (stderr, " WIN32 Release\n");
 332 #endif
 333 #endif
 334 #if HAVE_BZLIB_H
 335             fprintf (stderr, "libbzip2\n"
 336                      " (C) 1996-1999 Julian R Seward.  All rights reserved.\n");
 337 #endif
 338         }
 339         else if (ret == 'v')
 340             yaz_log_init_level (yaz_log_mask_str(arg));
 341         else if (ret == 'l')
 342             yaz_log_init_file (arg);
 343         else if (ret == 'm')
 344             mem_max = 1024*1024*atoi(arg);
 345         else if (ret == 'd')
 346             rGroupDef.databaseName = arg;
 347         else if (ret == 's')
 348             rGroupDef.flagRw = 0;
 349         else if (ret == 'g')
 350             rGroupDef.groupName = arg;
 351         else if (ret == 'f')
 352             rGroupDef.fileVerboseLimit = atoi(arg);
 353         else if (ret == 'c')
 354             configName = arg;
 355         else if (ret == 't')
 356             rGroupDef.recordType = arg;
 357         else if (ret == 'n')
 358             disableCommit = 1;
 359 #if ZEBRASDR
 360         else if (ret == 'S')
 361             rGroupDef.useSDR = 1;
 362 #endif
 363         else
 364             logf (LOG_WARN, "unknown option '-%s'", arg);
 365     }
 366     recTypes_destroy (rGroupDef.recTypes);
 367     if (common_resource)
 368     {
 369         zebraIndexUnlock ();
 370         bfs_destroy (rGroupDef.bfs);
 371     }
 372     data1_destroy (rGroupDef.dh);
 373     exit (0);
 374     return 0;
 375 }
 376