From 83762ea76e5af65ccb4407c6b38053bc6491a875 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 20 Nov 1995 11:56:21 +0000 Subject: [PATCH] Work on new traversal. --- index/Makefile | 22 +++--- index/dir.c | 38 ++++++++-- index/dirs.c | 212 +++++++++++++++++++++++++++++++++---------------------- index/extract.c | 16 +++-- index/index.h | 30 +++++++- index/main.c | 9 ++- index/trav.c | 159 +++++++++++++++++++++++++++++++++++++---- 7 files changed, 361 insertions(+), 125 deletions(-) diff --git a/index/Makefile b/index/Makefile index 9abdee1..673a076 100644 --- a/index/Makefile +++ b/index/Makefile @@ -1,15 +1,17 @@ # Copyright (C) 1995, Index Data I/S # All rights reserved. # Sebastian Hammer, Adam Dickmeiss -# $Id: Makefile,v 1.19 1995-11-17 15:54:41 adam Exp $ +# $Id: Makefile,v 1.20 1995-11-20 11:56:21 adam Exp $ SHELL=/bin/sh RANLIB=ranlib -YAZ=../../yaz -YAZLIB=$(YAZ)/lib/libyaz.a -OSILIB=../../xtimosi/src/libmosi.a $(YAZ)/lib/librfc.a + +YAZLIB=-lyaz +YAZINC= +OSILIB=../../xtimosi/src/libmosi.a -lrfc #NETLIB=-lnsl -lsocket -INCLUDE=-I../include -I$(YAZ)/include + +INCLUDE=-I../include $(YAZINC) TPROG1=index TPROG2=kdump TPROG3=zserver @@ -24,19 +26,17 @@ CPP=$(CC) -E all: $(TPROG1) $(TPROG2) $(TPROG3) $(TPROG1): $(O1) ../lib/dict.a \ - ../lib/isam.a ../lib/bfile.a ../lib/alexutil.a \ - $(YAZLIB) + ../lib/isam.a ../lib/bfile.a ../lib/alexutil.a $(CC) $(CFLAGS) -o $(TPROG1) $(O1) ../lib/dict.a \ ../lib/isam.a ../lib/bfile.a ../lib/alexutil.a \ - $(YAZLIB) $(OSILIB) + $(YAZLIB) -$(TPROG2): $(O2) $(YAZLIB) +$(TPROG2): $(O2) $(CC) $(CFLAGS) -o $(TPROG2) $(O2) $(YAZLIB) $(TPROG3): $(O3) \ ../lib/rset.a ../lib/dict.a ../lib/isam.a ../lib/bfile.a \ - ../lib/dfa.a ../lib/alexutil.a \ - $(YAZLIB) + ../lib/dfa.a ../lib/alexutil.a $(CC) $(CFLAGS) -o $(TPROG3) $(O3) \ ../lib/rset.a ../lib/dict.a ../lib/isam.a ../lib/bfile.a \ ../lib/dfa.a ../lib/alexutil.a \ diff --git a/index/dir.c b/index/dir.c index 66d0316..bc88bfb 100644 --- a/index/dir.c +++ b/index/dir.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: dir.c,v $ - * Revision 1.9 1995-10-30 13:42:12 adam + * Revision 1.10 1995-11-20 11:56:22 adam + * Work on new traversal. + * + * Revision 1.9 1995/10/30 13:42:12 adam * Added errno.h * * Revision 1.8 1995/10/10 13:59:23 adam @@ -34,6 +37,7 @@ * */ #include +#include #include #include #include @@ -49,6 +53,8 @@ struct dir_entry *dir_open (const char *rep) { DIR *dir; + char path[256]; + size_t pathpos; struct dirent *dent; size_t entry_max = 500; size_t idx = 0; @@ -58,13 +64,18 @@ struct dir_entry *dir_open (const char *rep) if (!(dir = opendir(rep))) { logf (LOG_WARN|LOG_ERRNO, "opendir %s", rep); - if (errno != ENOENT) + if (errno != ENOENT && errno != EACCES) exit (1); return NULL; } entry = xmalloc (sizeof(*entry) * entry_max); + strcpy (path, rep); + pathpos = strlen(path); + if (!pathpos || path[pathpos-1] != '/') + path[pathpos++] = '/'; while ((dent = readdir (dir))) { + struct stat finfo; if (strcmp (dent->d_name, ".") == 0 || strcmp (dent->d_name, "..") == 0) continue; @@ -78,9 +89,26 @@ struct dir_entry *dir_open (const char *rep) entry = entry_n; entry_max += 100; } - entry[idx].name = xmalloc (strlen(dent->d_name)+1); - strcpy (entry[idx].name, dent->d_name); - idx++; + strcpy (path + pathpos, dent->d_name); + stat (path, &finfo); + switch (finfo.st_mode & S_IFMT) + { + case S_IFREG: + entry[idx].kind = dirs_file; + entry[idx].ctime = finfo.st_ctime; + entry[idx].name = xmalloc (strlen(dent->d_name)+1); + strcpy (entry[idx].name, dent->d_name); + idx++; + break; + case S_IFDIR: + entry[idx].kind = dirs_dir; + entry[idx].ctime = finfo.st_ctime; + entry[idx].name = xmalloc (strlen(dent->d_name)+2); + strcpy (entry[idx].name, dent->d_name); + strcat (entry[idx].name, "/"); + idx++; + break; + } } entry[idx].name = NULL; closedir (dir); diff --git a/index/dirs.c b/index/dirs.c index d4460ba..ce6da3f 100644 --- a/index/dirs.c +++ b/index/dirs.c @@ -4,37 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: dirs.c,v $ - * Revision 1.1 1995-11-17 15:54:42 adam - * Started work on virtual directory structure. - * - * Revision 1.9 1995/10/30 13:42:12 adam - * Added errno.h - * - * Revision 1.8 1995/10/10 13:59:23 adam - * Function rset_open changed its wflag parameter to general flags. - * - * Revision 1.7 1995/09/28 09:19:40 adam - * xfree/xmalloc used everywhere. - * Extract/retrieve method seems to work for text records. - * - * Revision 1.6 1995/09/08 14:52:26 adam - * Minor changes. Dictionary is lower case now. - * - * Revision 1.5 1995/09/06 16:11:16 adam - * Option: only one word key per file. - * - * Revision 1.4 1995/09/04 12:33:41 adam - * Various cleanup. YAZ util used instead. - * - * Revision 1.3 1995/09/01 14:06:35 adam - * Split of work into more files. - * - * Revision 1.2 1995/09/01 10:57:07 adam - * Minor changes. - * - * Revision 1.1 1995/09/01 10:34:51 adam - * Added dir.c + * Revision 1.2 1995-11-20 11:56:23 adam + * Work on new traversal. * + * Revision 1.1 1995/11/17 15:54:42 adam + * Started work on virtual directory structure. */ #include #include @@ -45,90 +19,155 @@ #include #include "index.h" -struct dirs_entry { - char path[160]; - int sysno; -}; - struct dirs_info { - int no; - struct dirs_entry *entries; -}; - -struct dirs_client_info { - struct dirs_info *di; + Dict dict; + int no_read; + int no_cur; int no_max; - char *prefix; + struct dirs_entry *entries; + char nextpath[256]; + char prefix[256]; int prelen; + struct dirs_entry *last_entry; }; static int dirs_client_proc (Dict_char *name, const char *info, int pos, void *client) { - struct dirs_client_info *ci = client; + struct dirs_info *ci = client; + struct dirs_entry *entry; if (memcmp (name, ci->prefix, ci->prelen)) return 1; - if (ci->di->no == ci->no_max) + if (ci->no_cur < 0) { - if (ci->no_max > 0) - { - struct dirs_entry *arn; - - ci->no_max += 1000; - arn = malloc (sizeof(*arn) * (ci->no_max)); - if (!arn) - { - logf (LOG_FATAL|LOG_ERRNO, "malloc"); - exit (1); - } - memcpy (arn, ci->di->entries, ci->di->no * sizeof(*arn)); - free (ci->di->entries); - ci->di->entries = arn; - } + ci->no_cur = 0; + return 0; + } + if (ci->no_cur == ci->no_max) + { + assert (0); + } + entry = ci->entries + ci->no_cur; + if (info[0] == sizeof(entry->sysno)+sizeof(entry->ctime)) + { + strcpy (entry->path, name + ci->prelen); + entry->kind = dirs_file; + memcpy (&entry->sysno, info+1, sizeof(entry->sysno)); + memcpy (&entry->ctime, info+1+sizeof(entry->sysno), + sizeof(entry->ctime)); + ci->no_cur++; + } + else if (info[0] == sizeof(entry->ctime)) + { + strcpy (entry->path, name + ci->prelen); + entry->kind = dirs_dir; + memcpy (&entry->ctime, info+1, sizeof(entry->ctime)); + ci->no_cur++; } - strcpy ((ci->di->entries + ci->di->no)->path, name); - memcpy (&(ci->di->entries + ci->di->no)->sysno, info+1, *info); - assert (*info == sizeof(ci->di->entries->sysno)); - ++(ci->di->no); return 0; } struct dirs_info *dirs_open (Dict dict, const char *rep) { - static char wname[200]; - static char pname[200]; - int dirid; - char *dinfo; - struct dirs_client_info dirs_client_info; struct dirs_info *p; - int before, after; + int before = 0, after; - sprintf (wname, "d%s", rep); - dinfo = dict_lookup (dict, wname); - if (!dinfo) - return NULL; + logf (LOG_DEBUG, "dirs_open %s", rep); if (!(p = malloc (sizeof (*p)))) { logf (LOG_FATAL|LOG_ERRNO, "malloc"); exit (1); } - memcpy (&dirid, dinfo+1, sizeof(dirid)); - sprintf (wname, "%d.", dirid); - strcpy (pname, wname); - - p->no = 0; - dirs_client_info.di = p; - dirs_client_info.no_max = 0; - dirs_client_info.prefix = wname; - dirs_client_info.prelen = strlen(wname); - strcpy (pname, wname); - dict_scan (dict, pname, &before, &after, &dirs_client_info, - dirs_client_proc); - + p->dict = dict; + strcpy (p->prefix, rep); + p->prelen = strlen(p->prefix); + strcpy (p->nextpath, rep); + p->no_read = p->no_cur = 0; + after = p->no_max = 400; + if (!(p->entries = malloc (sizeof(*p->entries) * (p->no_max)))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + logf (LOG_DEBUG, "dirs_open first scan"); + dict_scan (p->dict, p->nextpath, &before, &after, p, dirs_client_proc); return p; } +struct dirs_entry *dirs_read (struct dirs_info *p) +{ + int before = 0, after = p->no_max; + + if (p->no_read < p->no_cur) + { + logf (LOG_DEBUG, "dirs_read %d. returns %s", p->no_read, + (p->entries + p->no_read)->path); + return p->last_entry = p->entries + (p->no_read++); + } + if (p->no_cur < p->no_max) + return p->last_entry = NULL; +#if 0 + strcpy (p->nextpath, p->prefix); + strcat (p->nextpath, (p->entries + p->no_max-1)->path); +#endif + p->no_cur = -1; + logf (LOG_DEBUG, "dirs_read rescan"); + dict_scan (p->dict, p->nextpath, &before, &after, p, dirs_client_proc); + p->no_read = 1; + if (p->no_read < p->no_cur) + return p->last_entry = p->entries; + return p->last_entry = NULL; +} + +struct dirs_entry *dirs_last (struct dirs_info *p) +{ + return p->last_entry; +} + +void dirs_mkdir (struct dirs_info *p, const char *src, int ctime) +{ + char path[256]; + + sprintf (path, "%s%s", p->prefix, src); + logf (LOG_DEBUG, "dirs_mkdir %s", path); + dict_insert (p->dict, path, sizeof(ctime), &ctime); +} + +void dirs_rmdir (struct dirs_info *p, const char *src) +{ + char path[256]; + char info[2]; + + sprintf (path, "%s%s", p->prefix, src); + logf (LOG_DEBUG, "dirs_rmdir %s", path); + info[0] = 'r'; + dict_insert (p->dict, path, 1, info); +} + +void dirs_add (struct dirs_info *p, const char *src, int sysno, int ctime) +{ + char path[256]; + char info[16]; + + sprintf (path, "%s%s", p->prefix, src); + logf (LOG_DEBUG, "dirs_add %s", path); + memcpy (info, &sysno, sizeof(sysno)); + memcpy (info+sizeof(sysno), &ctime, sizeof(ctime)); + dict_insert (p->dict, path, sizeof(sysno)+sizeof(ctime), info); +} + +void dirs_del (struct dirs_info *p, const char *src) +{ + char path[256]; + char info[2]; + + sprintf (path, "%s%s", p->prefix, src); + logf (LOG_DEBUG, "dirs_del %s", path); + info[0] = 'r'; + dict_insert (p->dict, path, 1, info); +} + void dirs_free (struct dirs_info **pp) { struct dirs_info *p = *pp; @@ -137,3 +176,4 @@ void dirs_free (struct dirs_info **pp) free (p); *pp = NULL; } + diff --git a/index/extract.c b/index/extract.c index 7b12e39..9207ab7 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.25 1995-11-16 15:34:54 adam + * Revision 1.26 1995-11-20 11:56:24 adam + * Work on new traversal. + * + * Revision 1.25 1995/11/16 15:34:54 adam * Uses new record management system in both indexer and server. * * Revision 1.24 1995/11/15 19:13:08 adam @@ -392,8 +395,8 @@ static int file_read (int fd, char *buf, size_t count) return read (fd, buf, count); } #endif -void file_extract (int cmd, const char *fname, const char *kname, - char *databaseName) +SYSNO file_extract (int cmd, const char *fname, const char *kname, + char *databaseName) { int i, r; char ext[128]; @@ -418,9 +421,9 @@ void file_extract (int cmd, const char *fname, const char *kname, } sprintf (ext_res, "fileExtension.%s", ext); if (!(file_type = res_get (common_resource, ext_res))) - return; + return 0; if (!(rt = recType_byName (file_type))) - return; + return 0; logf (LOG_DEBUG, "%c %s k=%s", cmd, fname, kname); file_info = dict_lookup (file_idx, kname); if (!file_info) @@ -448,7 +451,7 @@ void file_extract (int cmd, const char *fname, const char *kname, if ((extractCtrl.fd = open (fname, O_RDONLY)) == -1) { logf (LOG_WARN|LOG_ERRNO, "open %s", fname); - return; + return 0; } extractCtrl.subType = ""; extractCtrl.init = wordInit; @@ -466,4 +469,5 @@ void file_extract (int cmd, const char *fname, const char *kname, close (extractCtrl.fd); if (r) logf (LOG_WARN, "Couldn't extract file %s, code %d", fname, r); + return sysno; } diff --git a/index/index.h b/index/index.h index 9ebf430..7989608 100644 --- a/index/index.h +++ b/index/index.h @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: index.h,v $ - * Revision 1.21 1995-11-16 15:34:55 adam + * Revision 1.22 1995-11-20 11:56:26 adam + * Work on new traversal. + * + * Revision 1.21 1995/11/16 15:34:55 adam * Uses new record management system in both indexer and server. * * Revision 1.20 1995/11/15 14:46:18 adam @@ -85,18 +88,39 @@ struct it_key { int seqno; }; +enum dirsKind { dirs_dir, dirs_file }; + struct dir_entry { + enum dirsKind kind; char *name; + int ctime; }; +struct dirs_entry { + enum dirsKind kind; + char path[256]; + int sysno; + int ctime; +}; + +struct dirs_info *dirs_open (Dict dict, const char *rep); +struct dirs_entry *dirs_read (struct dirs_info *p); +struct dirs_entry *dirs_last (struct dirs_info *p); +void dirs_mkdir (struct dirs_info *p, const char *src, int ctime); +void dirs_rmdir (struct dirs_info *p, const char *src); +void dirs_add (struct dirs_info *p, const char *src, int sysno, int ctime); +void dirs_del (struct dirs_info *p, const char *src); +void dirs_free (struct dirs_info **pp); + struct dir_entry *dir_open (const char *rep); void dir_sort (struct dir_entry *e); void dir_free (struct dir_entry **e_p); void repository (int cmd, const char *rep, const char *base_path, char *databaseName); +void repositoryUpdate (const char *path, char *databaseName); -void file_extract (int cmd, const char *fname, const char *kname, - char *databaseName); +SYSNO file_extract (int cmd, const char *fname, const char *kname, + char *databaseName); void key_open (int mem); int key_close (void); diff --git a/index/main.c b/index/main.c index cb73771..92ea245 100644 --- a/index/main.c +++ b/index/main.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: main.c,v $ - * Revision 1.15 1995-11-01 16:25:51 quinn + * Revision 1.16 1995-11-20 11:56:27 adam + * Work on new traversal. + * + * Revision 1.15 1995/11/01 16:25:51 quinn * *** empty log message *** * * Revision 1.14 1995/10/17 18:02:09 adam @@ -114,7 +117,11 @@ int main (int argc, char **argv) key_open (mem_max); key_open_flag = 1; } +#if 0 repository (cmd, arg, base_path, databaseName); +#else + repositoryUpdate (arg, databaseName); +#endif cmd = 0; } } diff --git a/index/trav.c b/index/trav.c index f2f4f35..faf7571 100644 --- a/index/trav.c +++ b/index/trav.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: trav.c,v $ - * Revision 1.6 1995-11-17 15:54:42 adam + * Revision 1.7 1995-11-20 11:56:28 adam + * Work on new traversal. + * + * Revision 1.6 1995/11/17 15:54:42 adam * Started work on virtual directory structure. * * Revision 1.5 1995/10/17 18:02:09 adam @@ -287,6 +290,148 @@ void repository_update_r (int cmd, char *dst, char *src, char *databaseName) dir_free (&e_src); } +static int repComp (const char *a, const char *b, size_t len) +{ + if (!len) + return 0; + return memcmp (a, b, len); +} + +static void repositoryUpdateR (struct dirs_info *di, struct dirs_entry *dst, + const char *base, char *src, char *databaseName) +{ + struct dir_entry *e_src; + int i_src = 0; + static char tmppath[256]; + size_t src_len = strlen (src); + + sprintf (tmppath, "%s%s", base, src); + e_src = dir_open (tmppath); + +#if 1 + if (!dst || repComp (dst->path, src, src_len)) +#else + if (!dst || strcmp (dst->path, src)) +#endif + { + if (!e_src) + return; +#if 1 + if (src_len && src[src_len-1] == '/') + --src_len; + else + src[src_len] = '/'; + src[src_len+1] = '\0'; +#endif + dirs_mkdir (di, src, 0); + dst = NULL; + } + else if (!e_src) + { + /* delete tree dst */ + return; + } + else + { +#if 1 + if (src_len && src[src_len-1] == '/') + --src_len; + else + src[src_len] = '/'; + src[src_len+1] = '\0'; +#endif + dst = dirs_read (di); + } + dir_sort (e_src); + + while (1) + { + int sd; + + if (dst && !repComp (dst->path, src, src_len)) + { + if (e_src[i_src].name) + { + logf (LOG_DEBUG, "dst=%s src=%s", dst->path + src_len+1, + e_src[i_src].name); + sd = strcmp (dst->path + src_len+1, e_src[i_src].name); + } + else + sd = -1; + } + else if (e_src[i_src].name) + sd = 1; + else + break; + logf (LOG_DEBUG, "trav sd=%d", sd); + if (sd == 0) + { + strcpy (src + src_len+1, e_src[i_src].name); + sprintf (tmppath, "%s%s", base, src); + + switch (e_src[i_src].kind) + { + case dirs_file: + if (e_src[i_src].ctime > dst->ctime) + { + file_extract ('d', tmppath, tmppath, databaseName); + file_extract ('a', tmppath, tmppath, databaseName); + dirs_add (di, src, dst->sysno, e_src[i_src].ctime); + } + dst = dirs_read (di); + break; + case dirs_dir: + repositoryUpdateR (di, dst, base, src, databaseName); + dst = dirs_last (di); + logf (LOG_DEBUG, "last is %s", dst ? dst->path : "null"); + break; + default: + dst = dirs_read (di); + } + i_src++; + } + else if (sd > 0) + { + SYSNO sysno; + strcpy (src + src_len+1, e_src[i_src].name); + sprintf (tmppath, "%s%s", base, src); + + switch (e_src[i_src].kind) + { + case dirs_file: + sysno = file_extract ('a', tmppath, tmppath, databaseName); + dirs_add (di, src, sysno, e_src[i_src].ctime); + break; + case dirs_dir: + repositoryUpdateR (di, dst, base, src, databaseName); + break; + } + i_src++; + } + else /* sd < 0 */ + { + assert (0); + } + } + dir_free (&e_src); +} + +void repositoryUpdate (const char *path, char *databaseName) +{ + struct dirs_info *di; + char src[256]; + Dict dict; + + dict = dict_open ("repdict", 40, 1); + + di = dirs_open (dict, path); + strcpy (src, ""); + repositoryUpdateR (di, dirs_read (di), path, src, databaseName); + dirs_free (&di); + + dict_close (dict); +} + void repository (int cmd, const char *rep, const char *base_path, char *databaseName) { @@ -303,15 +448,3 @@ void repository (int cmd, const char *rep, const char *base_path, repository_extract_r (cmd, rep_tmp1, databaseName); } -void repositoryUpdate (const char *src, char *databaseName) -{ - struct dir_entry *e_src; - int i_src = 0; - struct stat fs_src; - size_t src_len = strlen (src); - - e_src = dir_open (src); - - if (!e_src) - return; -} -- 1.7.10.4