From 589b5761fb59634afc37909e62cb34ce56cbbb12 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 5 Feb 1996 12:29:55 +0000 Subject: [PATCH] Logging reduced a bit. The remaining running time is estimated during register merge. --- index/dir.c | 10 ++++--- index/extract.c | 62 +++++++++++++++++++++++++++++++++++------- index/kinput.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++---- index/trav.c | 16 ++++++----- 4 files changed, 144 insertions(+), 24 deletions(-) diff --git a/index/dir.c b/index/dir.c index cd80bff..4ed2885 100644 --- a/index/dir.c +++ b/index/dir.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: dir.c,v $ - * Revision 1.11 1995-11-20 16:59:44 adam + * Revision 1.12 1996-02-05 12:29:55 adam + * Logging reduced a bit. + * The remaining running time is estimated during register merge. + * + * Revision 1.11 1995/11/20 16:59:44 adam * New update method: the 'old' keys are saved for each records. * * Revision 1.10 1995/11/20 11:56:22 adam @@ -63,7 +67,7 @@ struct dir_entry *dir_open (const char *rep) size_t idx = 0; struct dir_entry *entry; - logf (LOG_LOG, "dir_open %s", rep); + logf (LOG_DEBUG, "dir_open %s", rep); if (!(dir = opendir(rep))) { logf (LOG_WARN|LOG_ERRNO, "opendir %s", rep); @@ -114,7 +118,7 @@ struct dir_entry *dir_open (const char *rep) } entry[idx].name = NULL; closedir (dir); - logf (LOG_LOG, "dir_close"); + logf (LOG_DEBUG, "dir_close"); return entry; } diff --git a/index/extract.c b/index/extract.c index 4a5e2ec..798354a 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.48 1996-02-01 20:53:26 adam + * Revision 1.49 1996-02-05 12:29:57 adam + * Logging reduced a bit. + * The remaining running time is estimated during register merge. + * + * Revision 1.48 1996/02/01 20:53:26 adam * The temporary per-record keys are compacted a little, and duplication * of the per-records keys are avoided when they are saved in the record * information buffer. @@ -696,8 +700,8 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup, } if (first) { - logf (LOG_WARN, "Record in file %s didn't contain match" - " fields in (%d,%d)", fname, attrSet, attrUse); + logf (LOG_WARN, "Record didn't contain match" + " fields in (%d,%d)", attrSet, attrUse); return NULL; } } @@ -767,6 +771,25 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup, return dstBuf; } +struct recordLogInfo { + const char *fname; + char *op; + struct recordGroup *rGroup; +}; + +static void recordLogPreamble (int level, const char *msg, void *info) +{ + struct recordLogInfo *p = info; + FILE *outf = log_file (); + + if (level & LOG_LOG) + return ; + if (p->op) + fprintf (outf, "%s of ", p->op); + fprintf (outf, "%s type %s\n", p->rGroup->recordType, p->fname); + log_event_start (NULL, NULL); +} + static int recordExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, int deleteFlag, struct file_read_info *fi, RecType recType, @@ -777,7 +800,13 @@ static int recordExtract (SYSNO *sysno, const char *fname, char *matchStr; SYSNO sysnotmp; Record rec; + struct recordLogInfo logInfo; + logInfo.fname = fname; + logInfo.op = NULL; + logInfo.rGroup = rGroup; + log_event_start (recordLogPreamble, &logInfo); + if (fi->fd != -1) { extractCtrl.fh = fi; @@ -820,7 +849,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, } else { - logf (LOG_WARN, "Record not inserted"); + logf (LOG_WARN, "Bad match criteria"); return 0; } } @@ -831,10 +860,14 @@ static int recordExtract (SYSNO *sysno, const char *fname, { if (deleteFlag) { - logf (LOG_LOG, "? %s", fname); + logf (LOG_LOG, "Cannot delete new record"); return 1; } - logf (LOG_LOG, "add %s %s", rGroup->recordType, fname); + logInfo.op = "add"; +#if 0 + logf (LOG_LOG, "update %s %s", rGroup->recordType, + fname); +#endif rec = rec_new (records); *sysno = rec->sysno; @@ -857,14 +890,16 @@ static int recordExtract (SYSNO *sysno, const char *fname, flushRecordKeys (*sysno, 0, &delkeys, rec->info[recInfo_databaseName]); if (deleteFlag) { + logInfo.op = "delete"; if (!delkeys.buf_used) { - logf (LOG_WARN, "cannot delete %s: storeKeys false", - fname); + logf (LOG_WARN, "cannot delete; storeKeys false"); } else { +#if 0 logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname); +#endif records_deleted++; if (matchStr) dict_delete (matchDict, matchStr); @@ -874,15 +909,17 @@ static int recordExtract (SYSNO *sysno, const char *fname, } else { + logInfo.op = "update"; if (!delkeys.buf_used) { - logf (LOG_WARN, "cannot update %s: storeKeys false", - fname); + logf (LOG_WARN, "cannot update; storeKeys false"); } else { +#if 0 logf (LOG_LOG, "update %s %s", rGroup->recordType, fname); +#endif flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); records_updated++; } @@ -953,6 +990,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); rec_put (records, &rec); + log_event_start (NULL, NULL); return 1; } @@ -999,14 +1037,18 @@ int fileExtract (SYSNO *sysno, const char *fname, sprintf (ext_res, "%srecordType", gprefix); if (!(rGroup->recordType = res_get (common_resource, ext_res))) { +#if 0 logf (LOG_LOG, "? %s", fname); +#endif return 0; } } } if (!rGroup->recordType) { +#if 0 logf (LOG_LOG, "? record %s", fname); +#endif return 0; } if (!(recType = recType_byName (rGroup->recordType, subType))) diff --git a/index/kinput.c b/index/kinput.c index b263fc7..23687bb 100644 --- a/index/kinput.c +++ b/index/kinput.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: kinput.c,v $ - * Revision 1.12 1995-12-06 17:49:19 adam + * Revision 1.13 1996-02-05 12:30:00 adam + * Logging reduced a bit. + * The remaining running time is estimated during register merge. + * + * Revision 1.12 1995/12/06 17:49:19 adam * Uses dict_delete now. * * Revision 1.11 1995/12/06 16:06:43 adam @@ -50,6 +54,7 @@ #include #include #include +#include #include #include #include @@ -78,20 +83,36 @@ struct key_file { char *prev_name; /* last word read */ int sysno; /* last sysno */ int seqno; /* last seqno */ + off_t length; /* length of file */ + /* handler invoked in each read */ + void (*readHandler)(struct key_file *keyp, void *rinfo); + void *readInfo; }; +void getFnameTmp (char *fname, int no) +{ + sprintf (fname, TEMP_FNAME, no); +} + void key_file_chunk_read (struct key_file *f) { int nr = 0, r, fd; char fname[256]; - sprintf (fname, TEMP_FNAME, f->no); + getFnameTmp (fname, f->no); fd = open (fname, O_RDONLY); if (fd == -1) { logf (LOG_FATAL|LOG_ERRNO, "cannot open %s", fname); exit (1); } - logf (LOG_LOG, "reading chunk from %s", fname); + if (!f->length) + { + if ((f->length = lseek (fd, 0L, SEEK_END)) == (off_t) -1) + { + logf (LOG_FATAL|LOG_ERRNO, "cannot seek %s", fname); + exit (1); + } + } if (lseek (fd, f->offset, SEEK_SET) == -1) { logf (LOG_FATAL|LOG_ERRNO, "cannot seek %s", fname); @@ -111,6 +132,8 @@ void key_file_chunk_read (struct key_file *f) } f->buf_size = nr; f->buf_ptr = 0; + if (f->readHandler) + (*f->readHandler)(f, f->readInfo); close (fd); } @@ -124,6 +147,8 @@ struct key_file *key_file_init (int no, int chunk) f->no = no; f->chunk = chunk; f->offset = 0; + f->length = 0; + f->readHandler = NULL; f->buf = xmalloc (f->chunk); f->prev_name = xmalloc (256); *f->prev_name = '\0'; @@ -385,6 +410,36 @@ int heap_inp (Dict dict, ISAM isam, struct heap_info *hi) return 0; } +struct progressInfo { + time_t startTime; + time_t lastTime; + off_t totalBytes; + off_t totalOffset; +}; + +void progressFunc (struct key_file *keyp, void *info) +{ + struct progressInfo *p = info; + time_t now, remaining; + + if (keyp->buf_size <= 0 || p->totalBytes <= 0) + return ; + p->totalOffset += keyp->buf_size; + time (&now); + + if (now < p->lastTime+10) + return ; + p->lastTime = now; + remaining = (now - p->startTime)* + ((double) p->totalBytes/p->totalOffset - 1.0); + if (remaining <= 130) + logf (LOG_LOG, "Merge %2.1f%% completed; %ld seconds remaining", + (100.0*p->totalOffset) / p->totalBytes, (long) remaining); + else + logf (LOG_LOG, "Merge %2.1f%% completed; %ld minutes remaining", + (100.0*p->totalOffset) / p->totalBytes, (long) remaining/60); +} + void key_input (const char *dict_fname, const char *isam_fname, int nkeys, int cache) @@ -395,6 +450,7 @@ void key_input (const char *dict_fname, const char *isam_fname, char rbuf[1024]; int i, r; struct heap_info *hi; + struct progressInfo progressInfo; dict = dict_open (dict_fname, cache, 1); if (!dict) @@ -410,9 +466,18 @@ void key_input (const char *dict_fname, const char *isam_fname, } kf = xmalloc ((1+nkeys) * sizeof(*kf)); + progressInfo.totalBytes = 0; + progressInfo.totalOffset = 0; + time (&progressInfo.startTime); + time (&progressInfo.lastTime); for (i = 1; i<=nkeys; i++) + { kf[i] = key_file_init (i, 32768); - + kf[i]->readHandler = progressFunc; + kf[i]->readInfo = &progressInfo; + progressInfo.totalBytes += kf[i]->length; + progressInfo.totalOffset += kf[i]->buf_size; + } hi = key_heap_init (nkeys, key_qsort_compare); for (i = 1; i<=nkeys; i++) if ((r = key_file_read (kf[i], rbuf))) @@ -421,7 +486,12 @@ void key_input (const char *dict_fname, const char *isam_fname, dict_close (dict); is_close (isam); - + + for (i = 1; i<=nkeys; i++) + { + getFnameTmp (rbuf, i); + unlink (rbuf); + } logf (LOG_LOG, "Iterations . . .%7d", no_iterations); logf (LOG_LOG, "Distinct words .%7d", no_diffs); logf (LOG_LOG, "Updates. . . . .%7d", no_updates); diff --git a/index/trav.c b/index/trav.c index 78746c4..454cf7f 100644 --- a/index/trav.c +++ b/index/trav.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: trav.c,v $ - * Revision 1.15 1995-12-07 17:38:48 adam + * Revision 1.16 1996-02-05 12:30:02 adam + * Logging reduced a bit. + * The remaining running time is estimated during register merge. + * + * Revision 1.15 1995/12/07 17:38:48 adam * Work locking mechanisms for concurrent updates/commit. * * Revision 1.14 1995/12/06 12:41:26 adam @@ -90,6 +94,7 @@ static void repositoryExtractR (int deleteFlag, char *rep, e = dir_open (rep); if (!e) return; + logf (LOG_LOG, "Dir: %s", rep); if (rep[rep_len-1] != '/') rep[rep_len] = '/'; else @@ -113,7 +118,7 @@ static void repositoryExtractR (int deleteFlag, char *rep, static void stdinExtractR (int deleteFlag, struct recordGroup *rGroup) { - char tmppath[256]; + char tmppath[1024]; logf (LOG_LOG, "stdinExtractR"); while (scanf ("%s", tmppath) == 1) @@ -124,7 +129,7 @@ static void repositoryDeleteR (struct dirs_info *di, struct dirs_entry *dst, const char *base, char *src, struct recordGroup *rGroup) { - char tmppath[256]; + char tmppath[1024]; size_t src_len = strlen (src); while (dst && !repComp (dst->path, src, src_len+1)) @@ -156,12 +161,12 @@ static void repositoryUpdateR (struct dirs_info *di, struct dirs_entry *dst, { struct dir_entry *e_src; int i_src = 0; - static char tmppath[256]; + static char tmppath[1024]; size_t src_len = strlen (src); sprintf (tmppath, "%s%s", base, src); e_src = dir_open (tmppath); - + logf (LOG_LOG, "Dir: %s", tmppath); #if 1 if (!dst || repComp (dst->path, src, src_len)) #else @@ -227,7 +232,6 @@ static void repositoryUpdateR (struct dirs_info *di, struct dirs_entry *dst, { if (fileExtract (&dst->sysno, tmppath, rGroup, 0)) { - logf (LOG_LOG, "dirs_add"); dirs_add (di, src, dst->sysno, e_src[i_src].ctime); } } -- 1.7.10.4