2 * Copyright (C) 1994-1999, Index Data
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.38 2002-04-04 14:14:13 adam
8 * Multiple registers (alpha early)
10 * Revision 1.37 2002/02/20 17:30:01 adam
11 * Work on new API. Locking system re-implemented
13 * Revision 1.36 1999/05/15 14:36:38 adam
14 * Updated dictionary. Implemented "compression" of dictionary.
16 * Revision 1.35 1999/02/02 14:51:09 adam
17 * Updated WIN32 code specific sections. Changed header.
19 * Revision 1.34 1998/06/08 14:43:14 adam
20 * Added suport for EXPLAIN Proxy servers - added settings databasePath
21 * and explainDatabase to facilitate this. Increased maximum number
22 * of databases and attributes in one register.
24 * Revision 1.33 1998/01/12 15:04:08 adam
25 * The test option (-s) only uses read-lock (and not write lock).
27 * Revision 1.32 1997/09/25 14:56:51 adam
28 * Windows NT interface code to the stat call.
30 * Revision 1.31 1997/09/17 12:19:17 adam
31 * Zebra version corresponds to YAZ version 1.4.
32 * Changed Zebra server so that it doesn't depend on global common_resource.
34 * Revision 1.30 1997/09/09 13:38:09 adam
35 * Partial port to WIN95/NT.
37 * Revision 1.29 1997/02/12 20:39:47 adam
38 * Implemented options -f <n> that limits the log to the first <n>
40 * Changed some log messages also.
42 * Revision 1.28 1996/11/01 08:58:44 adam
43 * Interface to isamc system now includes update and delete.
45 * Revision 1.27 1996/10/29 14:06:56 adam
46 * Include zebrautl.h instead of alexutil.h.
48 * Revision 1.26 1996/06/04 10:19:01 adam
49 * Minor changes - removed include of ctype.h.
51 * Revision 1.25 1996/05/01 13:46:37 adam
52 * First work on multiple records in one file.
53 * New option, -offset, to the "unread" command in the filter module.
55 * Revision 1.24 1996/04/26 10:00:23 adam
56 * Added option -V to zebraidx to display version information.
57 * Removed stupid warnings from file update.
59 * Revision 1.23 1996/04/12 07:02:25 adam
60 * File update of single files.
62 * Revision 1.22 1996/04/09 06:50:50 adam
63 * Bug fix: bad reference in function fileUpdateR.
65 * Revision 1.21 1996/03/22 15:34:18 quinn
68 * Revision 1.20 1996/03/21 14:50:10 adam
69 * File update uses modify-time instead of change-time.
71 * Revision 1.19 1996/03/20 16:16:55 quinn
72 * Added diagnostic output
74 * Revision 1.18 1996/03/19 12:43:27 adam
75 * Bug fix: File update traversal didn't handle trailing slashes correctly.
76 * Bug fix: Update of sub directory groups wasn't handled correctly.
78 * Revision 1.17 1996/02/12 18:45:17 adam
79 * Changed naming of some functions.
81 * Revision 1.16 1996/02/05 12:30:02 adam
82 * Logging reduced a bit.
83 * The remaining running time is estimated during register merge.
85 * Revision 1.15 1995/12/07 17:38:48 adam
86 * Work locking mechanisms for concurrent updates/commit.
88 * Revision 1.14 1995/12/06 12:41:26 adam
89 * New command 'stat' for the index program.
90 * Filenames can be read from stdin by specifying '-'.
91 * Bug fix/enhancement of the transformation from terms to regular
92 * expressons in the search engine.
94 * Revision 1.13 1995/11/28 09:09:46 adam
95 * Zebra config renamed.
96 * Use setting 'recordId' to identify record now.
97 * Bug fix in recindex.c: rec_release_blocks was invokeded even
98 * though the blocks were already released.
99 * File traversal properly deletes records when needed.
101 * Revision 1.12 1995/11/24 11:31:37 adam
102 * Commands add & del read filenames from stdin if source directory is
104 * Match criteria supports 'constant' strings.
106 * Revision 1.11 1995/11/22 17:19:19 adam
107 * Record management uses the bfile system.
109 * Revision 1.10 1995/11/21 15:01:16 adam
110 * New general match criteria implemented.
111 * New feature: document groups.
113 * Revision 1.9 1995/11/21 09:20:32 adam
114 * Yet more work on record match.
116 * Revision 1.8 1995/11/20 16:59:46 adam
117 * New update method: the 'old' keys are saved for each records.
119 * Revision 1.7 1995/11/20 11:56:28 adam
120 * Work on new traversal.
122 * Revision 1.6 1995/11/17 15:54:42 adam
123 * Started work on virtual directory structure.
125 * Revision 1.5 1995/10/17 18:02:09 adam
126 * New feature: databases. Implemented as prefix to words in dictionary.
128 * Revision 1.4 1995/09/28 09:19:46 adam
129 * xfree/xmalloc used everywhere.
130 * Extract/retrieve method seems to work for text records.
132 * Revision 1.3 1995/09/06 16:11:18 adam
133 * Option: only one word key per file.
135 * Revision 1.2 1995/09/04 12:33:43 adam
136 * Various cleanup. YAZ util used instead.
138 * Revision 1.1 1995/09/01 14:06:36 adam
139 * Split of work into more files.
146 #include <sys/types.h>
147 #include <sys/stat.h>
150 #define S_ISREG(x) (x & _S_IFREG)
151 #define S_ISDIR(x) (x & _S_IFDIR)
161 static int repComp (const char *a, const char *b, size_t len)
165 return memcmp (a, b, len);
168 static void repositoryExtractR (ZebraHandle zh, int deleteFlag, char *rep,
169 struct recordGroup *rGroup,
174 size_t rep_len = strlen (rep);
179 logf (LOG_LOG, "dir %s", rep);
180 if (rep[rep_len-1] != '/')
185 for (i=0; e[i].name; i++)
188 strcpy (rep +rep_len+1, e[i].name);
189 if ((ecp = strrchr (e[i].name, '/')))
191 if (level == 0 && rGroup->databaseNamePath)
192 rGroup->databaseName = e[i].name;
197 fileExtract (zh, NULL, rep, rGroup, deleteFlag);
200 repositoryExtractR (zh, deleteFlag, rep, rGroup, level+1);
208 static void fileDeleteR (ZebraHandle zh,
209 struct dirs_info *di, struct dirs_entry *dst,
210 const char *base, char *src,
211 struct recordGroup *rGroup)
214 size_t src_len = strlen (src);
216 while (dst && !repComp (dst->path, src, src_len+1))
221 sprintf (tmppath, "%s%s", base, dst->path);
222 fileExtract (zh, &dst->sysno, tmppath, rGroup, 1);
224 strcpy (tmppath, dst->path);
225 dst = dirs_read (di);
226 dirs_del (di, tmppath);
229 strcpy (tmppath, dst->path);
230 dst = dirs_read (di);
231 dirs_rmdir (di, tmppath);
234 dst = dirs_read (di);
239 static void fileUpdateR (ZebraHandle zh,
240 struct dirs_info *di, struct dirs_entry *dst,
241 const char *base, char *src,
242 struct recordGroup *rGroup,
245 struct dir_entry *e_src;
247 static char tmppath[1024];
248 size_t src_len = strlen (src);
250 sprintf (tmppath, "%s%s", base, src);
251 e_src = dir_open (tmppath);
252 logf (LOG_LOG, "dir %s", tmppath);
255 if (!dst || repComp (dst->path, src, src_len))
257 if (!dst || strcmp (dst->path, src))
263 if (src_len && src[src_len-1] != '/')
266 src[++src_len] = '\0';
268 dirs_mkdir (di, src, 0);
269 if (dst && repComp (dst->path, src, src_len))
274 strcpy (src, dst->path);
275 fileDeleteR (zh, di, dst, base, src, rGroup);
280 if (src_len && src[src_len-1] != '/')
283 src[++src_len] = '\0';
285 dst = dirs_read (di);
293 if (dst && !repComp (dst->path, src, src_len))
295 if (e_src[i_src].name)
297 logf (LOG_DEBUG, "dst=%s src=%s", dst->path + src_len,
299 sd = strcmp (dst->path + src_len, e_src[i_src].name);
304 else if (e_src[i_src].name)
308 logf (LOG_DEBUG, "trav sd=%d", sd);
310 if (level == 0 && rGroup->databaseNamePath)
311 rGroup->databaseName = e_src[i_src].name;
314 strcpy (src + src_len, e_src[i_src].name);
315 sprintf (tmppath, "%s%s", base, src);
317 switch (e_src[i_src].kind)
320 if (e_src[i_src].mtime > dst->mtime)
322 if (fileExtract (zh, &dst->sysno, tmppath, rGroup, 0))
324 dirs_add (di, src, dst->sysno, e_src[i_src].mtime);
326 logf (LOG_DEBUG, "old: %s", ctime (&dst->mtime));
327 logf (LOG_DEBUG, "new: %s", ctime (&e_src[i_src].mtime));
329 dst = dirs_read (di);
332 fileUpdateR (zh, di, dst, base, src, rGroup, level+1);
333 dst = dirs_last (di);
334 logf (LOG_DEBUG, "last is %s", dst ? dst->path : "null");
337 dst = dirs_read (di);
344 strcpy (src + src_len, e_src[i_src].name);
345 sprintf (tmppath, "%s%s", base, src);
347 switch (e_src[i_src].kind)
350 if (fileExtract (zh, &sysno, tmppath, rGroup, 0))
351 dirs_add (di, src, sysno, e_src[i_src].mtime);
354 fileUpdateR (zh, di, dst, base, src, rGroup, level+1);
356 dst = dirs_last (di);
363 strcpy (src, dst->path);
364 sprintf (tmppath, "%s%s", base, dst->path);
369 fileExtract (zh, &dst->sysno, tmppath, rGroup, 1);
370 dirs_del (di, dst->path);
371 dst = dirs_read (di);
374 fileDeleteR (zh, di, dst, base, src, rGroup);
375 dst = dirs_last (di);
382 static void groupRes (ZebraHandle zh, struct recordGroup *rGroup)
387 if (!rGroup->groupName || !*rGroup->groupName)
390 sprintf (gPrefix, "%s.", rGroup->groupName);
392 sprintf (resStr, "%srecordId", gPrefix);
393 rGroup->recordId = res_get (zh->res, resStr);
394 sprintf (resStr, "%sdatabasePath", gPrefix);
395 rGroup->databaseNamePath =
396 atoi (res_get_def (zh->res, resStr, "0"));
399 void repositoryShow (ZebraHandle zh)
402 struct recordGroup *rGroup = &zh->rGroup;
405 struct dirs_entry *dst;
407 struct dirs_info *di;
409 if (!(dict = dict_open (zh->reg->bfs, FMATCH_DICT, 50, 0, 0)))
411 logf (LOG_FATAL, "dict_open fail of %s", FMATCH_DICT);
415 assert (rGroup->path);
416 strcpy (src, rGroup->path);
417 src_len = strlen (src);
419 if (src_len && src[src_len-1] != '/')
422 src[++src_len] = '\0';
425 di = dirs_open (dict, src, rGroup->flagRw);
427 while ( (dst = dirs_read (di)) )
428 logf (LOG_LOG, "%s", dst->path);
433 static void fileUpdate (ZebraHandle zh,
434 Dict dict, struct recordGroup *rGroup,
437 struct dirs_info *di;
445 src_len = strlen (src);
448 if (S_ISREG(sbuf.st_mode))
450 struct dirs_entry *e_dst;
451 di = dirs_fopen (dict, src);
453 e_dst = dirs_read (di);
456 if (sbuf.st_mtime > e_dst->mtime)
457 if (fileExtract (zh, &e_dst->sysno, src, rGroup, 0))
458 dirs_add (di, src, e_dst->sysno, sbuf.st_mtime);
463 if (fileExtract (zh, &sysno, src, rGroup, 0))
464 dirs_add (di, src, sysno, sbuf.st_mtime);
468 else if (S_ISDIR(sbuf.st_mode))
470 if (src_len && src[src_len-1] != '/')
473 src[++src_len] = '\0';
475 di = dirs_open (dict, src, rGroup->flagRw);
477 fileUpdateR (zh, di, dirs_read (di), src, dst, rGroup, 0);
482 logf (LOG_WARN, "Ignoring path %s", src);
487 static void repositoryExtract (ZebraHandle zh,
488 int deleteFlag, struct recordGroup *rGroup,
498 if (S_ISREG(sbuf.st_mode))
499 fileExtract (zh, NULL, src, rGroup, deleteFlag);
500 else if (S_ISDIR(sbuf.st_mode))
501 repositoryExtractR (zh, deleteFlag, src, rGroup, 0);
503 logf (LOG_WARN, "Ignoring path %s", src);
506 static void repositoryExtractG (ZebraHandle zh,
507 int deleteFlag, struct recordGroup *rGroup)
509 if (*rGroup->path == '\0' || !strcmp(rGroup->path, "-"))
513 while (scanf ("%s", src) == 1)
514 repositoryExtract (zh, deleteFlag, rGroup, src);
517 repositoryExtract (zh, deleteFlag, rGroup, rGroup->path);
520 void repositoryUpdate (ZebraHandle zh)
522 struct recordGroup *rGroup = &zh->rGroup;
523 groupRes (zh, rGroup);
524 assert (rGroup->path);
525 if (rGroup->recordId && !strcmp (rGroup->recordId, "file"))
528 if (!(dict = dict_open (zh->reg->bfs, FMATCH_DICT, 50,
531 logf (LOG_FATAL, "dict_open fail of %s", FMATCH_DICT);
534 if (*rGroup->path == '\0' || !strcmp(rGroup->path, "-"))
537 while (scanf ("%s", src) == 1)
538 fileUpdate (zh, dict, rGroup, src);
541 fileUpdate (zh, dict, rGroup, rGroup->path);
545 repositoryExtractG (zh, 0, rGroup);
548 void repositoryDelete (ZebraHandle zh)
550 repositoryExtractG (zh, 1, &zh->rGroup);