From: Adam Dickmeiss Date: Fri, 8 Dec 1995 16:22:53 +0000 (+0000) Subject: Work on update while servers are running. Three lock files introduced. X-Git-Tag: ZEBRA.1.0~601 X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=commitdiff_plain;h=bf27228e5485151ee52bfd87dd91b025544ab6b2 Work on update while servers are running. Three lock files introduced. The servers reload their registers when necessary, but they don't reestablish result sets yet. --- diff --git a/index/index.h b/index/index.h index 620ffcf..a6ad7d9 100644 --- a/index/index.h +++ b/index/index.h @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: index.h,v $ - * Revision 1.32 1995-12-07 17:38:46 adam + * Revision 1.33 1995-12-08 16:22:53 adam + * Work on update while servers are running. Three lock files introduced. + * The servers reload their registers when necessary, but they don't + * reestablish result sets yet. + * + * Revision 1.32 1995/12/07 17:38:46 adam * Work locking mechanisms for concurrent updates/commit. * * Revision 1.31 1995/12/06 12:41:22 adam @@ -205,13 +210,12 @@ void rec_prstat (void); void zebraLockPrefix (char *pathPrefix); -int zebraServerLock (void); -void zebraServerUnlock (void); - void zebraIndexLockMsg (const char *str); -void zebraIndexUnlock (int rw); -void zebraIndexLock (int rw); -void zebraIndexLockCommit (void); -int zebraServerLockGetState (void); - -#define FNAME_MAIN_LOCK "zebra.lock" +void zebraIndexUnlock (void); +void zebraIndexLock (int commitNow); +int zebraIndexWait (int commitPhase); + +#define FNAME_MAIN_LOCK "zebraidx.LCK" +#define FNAME_COMMIT_LOCK "zebracmt.LCK" +#define FNAME_ORG_LOCK "zebraorg.LCK" +#define FNAME_TOUCH_TIME "zebraidx.time" diff --git a/index/lockidx.c b/index/lockidx.c index ed7c7f8..75bec86 100644 --- a/index/lockidx.c +++ b/index/lockidx.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: lockidx.c,v $ - * Revision 1.1 1995-12-07 17:38:47 adam + * Revision 1.2 1995-12-08 16:22:54 adam + * Work on update while servers are running. Three lock files introduced. + * The servers reload their registers when necessary, but they don't + * reestablish result sets yet. + * + * Revision 1.1 1995/12/07 17:38:47 adam * Work locking mechanisms for concurrent updates/commit. * */ @@ -20,10 +25,72 @@ #include "index.h" static int lock_fd = -1; +static int server_lock_cmt = -1; +static int server_lock_org = -1; + +int zebraIndexWait (int commitPhase) +{ + char pathPrefix[1024]; + char path[1024]; + int fd; + + zebraLockPrefix (pathPrefix); + + if (server_lock_cmt == -1) + { + sprintf (path, "%s%s", FNAME_COMMIT_LOCK, pathPrefix); + if ((server_lock_cmt = open (path, O_CREAT|O_RDWR|O_SYNC, 0666)) + == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "create %s", path); + return -1; + } + } + else + flock (server_lock_cmt, LOCK_UN); + if (server_lock_org == -1) + { + sprintf (path, "%s%s", FNAME_ORG_LOCK, pathPrefix); + if ((server_lock_org = open (path, O_CREAT|O_RDWR|O_SYNC, 0666)) + == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "create %s", path); + return -1; + } + } + else + flock (server_lock_org, LOCK_UN); + if (commitPhase) + fd = server_lock_cmt; + else + fd = server_lock_org; + if (flock (fd, LOCK_EX|LOCK_NB) == -1) + { + if (errno != EWOULDBLOCK) + { + logf (LOG_FATAL|LOG_ERRNO, "flock"); + exit (1); + } + if (commitPhase) + logf (LOG_LOG, "Waiting for lock cmt"); + else + logf (LOG_LOG, "Waiting for lock org"); + if (flock (fd, LOCK_EX) == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "flock"); + exit (1); + } + } + flock (fd, LOCK_UN); + return 0; +} + void zebraIndexLockMsg (const char *str) { - int l, r; + char path[1024]; + char pathPrefix[1024]; + int l, r, fd; assert (lock_fd != -1); lseek (lock_fd, 0L, SEEK_SET); @@ -34,105 +101,115 @@ void zebraIndexLockMsg (const char *str) logf (LOG_FATAL|LOG_ERRNO, "write lock file"); exit (1); } + zebraLockPrefix (pathPrefix); + sprintf (path, "%s%s", pathPrefix, FNAME_TOUCH_TIME); + fd = creat (path, 0666); + close (fd); } -void zebraIndexUnlock (int rw) +void zebraIndexUnlock (void) { char path[1024]; char pathPrefix[1024]; - if (lock_fd == -1) - return; - zebraLockPrefix (pathPrefix); sprintf (path, "%s%s", pathPrefix, FNAME_MAIN_LOCK); unlink (path); } -void zebraIndexLock (int rw) +void zebraIndexLock (int commitNow) { char path[1024]; char pathPrefix[1024]; char buf[256]; int r; + if (lock_fd != -1) + return ; zebraLockPrefix (pathPrefix); sprintf (path, "%s%s", pathPrefix, FNAME_MAIN_LOCK); - if (rw) + while (1) { - while (1) + lock_fd = open (path, O_CREAT|O_RDWR|O_EXCL|O_SYNC, 0666); + if (lock_fd == -1) { - lock_fd = open (path, O_CREAT|O_RDWR|O_EXCL|O_SYNC, 0666); - if (lock_fd == -1) + lock_fd = open (path, O_RDONLY); + assert (lock_fd != -1); + if (flock (lock_fd, LOCK_EX|LOCK_NB) == -1) { - lock_fd = open (path, O_RDONLY); - assert (lock_fd != -1); - if (flock (lock_fd, LOCK_EX|LOCK_NB) == -1) + if (errno == EWOULDBLOCK) { - if (errno == EWOULDBLOCK) - { - logf (LOG_LOG, "Waiting for other index process"); - flock (lock_fd, LOCK_EX); - continue; - } - else - { - logf (LOG_FATAL|LOG_ERRNO, "flock %s", path); - exit (1); - } + logf (LOG_LOG, "Waiting for other index process"); + flock (lock_fd, LOCK_EX); + continue; } else { - logf (LOG_WARN, "Unlocked %s", path); - r = read (lock_fd, buf, 256); - if (r == 0) - { - logf (LOG_WARN, "Zero length %s", path); - unlink (path); - close (lock_fd); - continue; - } - else if (r == -1) - { - logf (LOG_FATAL|LOG_ERRNO, "read %s", path); - exit (1); - } - if (*buf == 'r') + logf (LOG_FATAL|LOG_ERRNO, "flock %s", path); + exit (1); + } + } + else + { + logf (LOG_WARN, "Unlocked %s", path); + r = read (lock_fd, buf, 256); + if (r == 0) + { + logf (LOG_WARN, "Zero length %s", path); + close (lock_fd); + unlink (path); + continue; + } + else if (r == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "read %s", path); + exit (1); + } + if (*buf == 'r') + { + logf (LOG_WARN, "Previous transaction didn't" + " reach commit"); + close (lock_fd); + bf_commitClean (); + unlink (path); + continue; + } + else if (*buf == 'd') + { + logf (LOG_WARN, "Commit file wan't deleted after commit"); + close (lock_fd); + bf_commitClean (); + unlink (path); + continue; + } + else if (*buf == 'w') + { + logf (LOG_WARN, "Your index may be inconsistent"); + exit (1); + } + else if (*buf == 'c') + { + if (commitNow) { - logf (LOG_WARN, "Previous transaction didn't" - " reach commit"); unlink (path); close (lock_fd); continue; } - else if (*buf == 'w') - { - logf (LOG_WARN, "Your index may be inconsistent"); - exit (1); - } - else if (*buf == 'c') - { - logf (LOG_FATAL, "Previous transaction didn't" - " finish commit. Restore now!"); - exit (1); - } - else - { - logf (LOG_FATAL, "Unknown id 0x%02x in %s", *buf, - path); - exit (1); - } + logf (LOG_FATAL, "Previous transaction didn't" + " finish commit. Commit now!"); + exit (1); + } + else + { + logf (LOG_FATAL, "Unknown id 0x%02x in %s", *buf, + path); + exit (1); } } - else - break; } - flock (lock_fd, LOCK_EX); + else + break; } -} - -void zebraIndexLockCommit (void) -{ - + flock (lock_fd, LOCK_EX); } diff --git a/index/locksrv.c b/index/locksrv.c index 517e680..535af27 100644 --- a/index/locksrv.c +++ b/index/locksrv.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: locksrv.c,v $ - * Revision 1.1 1995-12-07 17:38:47 adam + * Revision 1.2 1995-12-08 16:22:55 adam + * Work on update while servers are running. Three lock files introduced. + * The servers reload their registers when necessary, but they don't + * reestablish result sets yet. + * + * Revision 1.1 1995/12/07 17:38:47 adam * Work locking mechanisms for concurrent updates/commit. * */ @@ -12,84 +17,100 @@ #include #include #include +#include #include #include #include #include -#include "index.h" +#include "zserver.h" -static int server_lock_fd = -1; +static int server_lock_cmt = -1; +static int server_lock_org = -1; -int zebraServerLock (void) +int zebraServerLock (int commitPhase) { char pathPrefix[1024]; char path[1024]; zebraLockPrefix (pathPrefix); - assert (server_lock_fd == -1); - sprintf (path, "%szebrasrv.%ld", pathPrefix, (long) getpid()); - if ((server_lock_fd = open (path, O_CREAT|O_RDWR|O_SYNC|O_EXCL, 0666)) - == -1) + if (server_lock_cmt == -1) { - logf (LOG_WARN|LOG_ERRNO, "remove stale %s", path); - unlink (path); - if ((server_lock_fd = open (path, O_CREAT|O_RDWR|O_SYNC|O_EXCL, 0666)) + sprintf (path, "%s%s", FNAME_COMMIT_LOCK, pathPrefix); + if ((server_lock_cmt = open (path, O_CREAT|O_RDWR, 0666)) == -1) { logf (LOG_FATAL|LOG_ERRNO, "create %s", path); return -1; } - } - flock (server_lock_fd, LOCK_EX); + assert (server_lock_org == -1); + sprintf (path, "%s%s", FNAME_ORG_LOCK, pathPrefix); + if ((server_lock_org = open (path, O_CREAT|O_RDWR, 0666)) + == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "create %s", path); + return -1; + } + } + if (commitPhase) + { + logf (LOG_LOG, "Server locks org"); + flock (server_lock_org, LOCK_SH); + } + else + { + logf (LOG_LOG, "Server locks cmt"); + flock (server_lock_cmt, LOCK_SH); + } return 0; } -int zebraServerLockGetState (void) +void zebraServerUnlock (int commitPhase) +{ + if (server_lock_org == -1) + return; + if (commitPhase) + { + logf (LOG_LOG, "Server unlocks org"); + flock (server_lock_org, LOCK_UN); + } + else + { + logf (LOG_LOG, "Server unlocks cmt"); + flock (server_lock_cmt, LOCK_UN); + } +} + +int zebraServerLockGetState (time_t *timep) { char pathPrefix[1024]; char path[1024]; char buf[256]; int fd; + struct stat xstat; zebraLockPrefix (pathPrefix); + sprintf (path, "%s%s", pathPrefix, FNAME_TOUCH_TIME); + if (stat (path, &xstat) == -1) + *timep = 1; + else + *timep = xstat.st_ctime; + sprintf (path, "%s%s", pathPrefix, FNAME_MAIN_LOCK); fd = open (path, O_RDONLY); if (fd == -1) + { + *buf = 0; return 0; + } if (read (fd, buf, 2) == 0) + { + *buf = 0; return 0; + } close (fd); return *buf; } - -void zebraServerLockMsg (const char *str) -{ - int l, r; - - assert (server_lock_fd != -1); - lseek (server_lock_fd, 0L, SEEK_SET); - l = strlen(str); - r = write (server_lock_fd, str, l); - if (r != l) - { - logf (LOG_FATAL|LOG_ERRNO, "write server lock file"); - exit (1); - } -} - -void zebraServerUnlock (void) -{ - char pathPrefix[1024]; - char path[1024]; - - assert (server_lock_fd != -1); - zebraLockPrefix (pathPrefix); - flock (server_lock_fd, LOCK_UN); - sprintf (path, "%szebrasrv.%ld", pathPrefix, (long) getpid()); - unlink (path); -} - diff --git a/index/main.c b/index/main.c index 5375e09..f40cd03 100644 --- a/index/main.c +++ b/index/main.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: main.c,v $ - * Revision 1.27 1995-12-07 17:38:47 adam + * Revision 1.28 1995-12-08 16:22:56 adam + * Work on update while servers are running. Three lock files introduced. + * The servers reload their registers when necessary, but they don't + * reestablish result sets yet. + * + * Revision 1.27 1995/12/07 17:38:47 adam * Work locking mechanisms for concurrent updates/commit. * * Revision 1.26 1995/12/06 12:41:23 adam @@ -150,11 +155,11 @@ int main (int argc, char **argv) { if (ret == 0) { + const char *rval; if(cmd == 0) /* command */ { if (!common_resource) { - const char *rval; common_resource = res_open (configName ? configName : FNAME_CONFIG); if (!common_resource) @@ -164,16 +169,6 @@ int main (int argc, char **argv) exit (1); } data1_tabpath = res_get (common_resource, "profilePath"); - rval = res_get (common_resource, "commitEnable"); - - zebraIndexLock (1); - if (rval && atoi(rval)) - { - zebraIndexLockMsg ("r"); - bf_cache (); - } - else - zebraIndexLockMsg ("w"); } if (!strcmp (arg, "update")) cmd = 'u'; @@ -181,12 +176,41 @@ int main (int argc, char **argv) cmd = 'd'; else if (!strcmp (arg, "commit")) { - logf (LOG_LOG, "Commit"); - zebraIndexLockMsg ("c"); - bf_commit (); + zebraIndexLock (1); + rval = res_get (common_resource, "commitEnable"); + if (rval && atoi (rval)) + bf_cache (1); + + if (bf_commitExists ()) + { + logf (LOG_LOG, "Commit start"); + zebraIndexLockMsg ("c"); + zebraIndexWait (1); + logf (LOG_LOG, "Commit execute"); + bf_commitExec (); + zebraIndexLockMsg ("d"); + zebraIndexWait (0); + logf (LOG_LOG, "Commit clean"); + bf_commitClean (); + } + else + logf (LOG_LOG, "Nothing to commit"); } else if (!strcmp (arg, "stat") || !strcmp (arg, "status")) { + zebraIndexLock (0); + bf_cache (0); + rec_prstat (); + } + else if (!strcmp (arg, "cstat") || !strcmp (arg, "cstatus")) + { + zebraIndexLock (1); + rval = res_get (common_resource, "commitEnable"); + if (rval && atoi(rval)) + { + bf_cache (1); + zebraIndexLockMsg ("r"); + } rec_prstat (); } else @@ -199,6 +223,17 @@ int main (int argc, char **argv) { struct recordGroup rGroup; + zebraIndexLock (0); + rval = res_get (common_resource, "commitEnable"); + if (rval && atoi(rval)) + { + bf_cache (1); + zebraIndexLockMsg ("r"); + } + else + zebraIndexLockMsg ("w"); + zebraIndexWait (0); + memcpy (&rGroup, &rGroupDef, sizeof(rGroup)); key_open (mem_max); rGroup.path = arg; @@ -248,7 +283,7 @@ int main (int argc, char **argv) exit (1); } } - zebraIndexUnlock (1); + zebraIndexUnlock (); exit (0); } diff --git a/index/zserver.c b/index/zserver.c index c718b70..9013c55 100644 --- a/index/zserver.c +++ b/index/zserver.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zserver.c,v $ - * Revision 1.30 1995-12-07 17:38:48 adam + * Revision 1.31 1995-12-08 16:22:56 adam + * Work on update while servers are running. Three lock files introduced. + * The servers reload their registers when necessary, but they don't + * reestablish result sets yet. + * + * Revision 1.30 1995/12/07 17:38:48 adam * Work locking mechanisms for concurrent updates/commit. * * Revision 1.29 1995/12/04 14:22:32 adam @@ -119,9 +124,10 @@ ZServerInfo server_info; -static int register_check (ZServerInfo *zi) +static int register_lock (ZServerInfo *zi) { - int state = zebraServerLockGetState(); + time_t lastChange; + int state = zebraServerLockGetState(&lastChange); switch (state) { @@ -131,26 +137,43 @@ static int register_check (ZServerInfo *zi) default: state = 0; } + zebraServerLock (state); if (zi->registerState == state) - return 0; - - zi->registerState = state; - if (server_info.records) { - dict_close (server_info.wordDict); - is_close (server_info.wordIsam); - rec_close (&server_info.records); + if (zi->registerChange >= lastChange) + return 0; + logf (LOG_LOG, "Register completely updated since last access"); } - /* enable commit if state is 1 */ - server_info.records = rec_open (0); - if (!(server_info.wordDict = dict_open (FNAME_WORD_DICT, 40, 0))) + else if (zi->registerState == -1) + logf (LOG_LOG, "Reading register using state %d pid=%ld", state, + (long) getpid()); + else + logf (LOG_LOG, "Register has changed state from %d to %d", + zi->registerState, state); + zi->registerChange = lastChange; + if (zi->records) + { + dict_close (zi->wordDict); + is_close (zi->wordIsam); + rec_close (&zi->records); + } + bf_cache (state); + zi->registerState = state; + zi->records = rec_open (0); + if (!(zi->wordDict = dict_open (FNAME_WORD_DICT, 40, 0))) return -1; - if (!(server_info.wordIsam = is_open (FNAME_WORD_ISAM, key_compare, 0, - sizeof (struct it_key)))) + if (!(zi->wordIsam = is_open (FNAME_WORD_ISAM, key_compare, 0, + sizeof (struct it_key)))) return -1; return 0; } +static int register_unlock (ZServerInfo *zi) +{ + if (zi->registerState != -1) + zebraServerUnlock (zi->registerState); +} + bend_initresult *bend_init (bend_initrequest *q) { static bend_initresult r; @@ -174,34 +197,14 @@ bend_initresult *bend_init (bend_initrequest *q) exit (1); } } - zebraServerLock (); - data1_tabpath = res_get(common_resource, "profilePath"); server_info.sets = NULL; server_info.registerState = -1; /* trigger open of registers! */ -#if 1 + server_info.registerChange = 0; + server_info.records = NULL; server_info.wordDict = NULL; server_info.wordIsam = NULL; -#else - server_info.records = rec_open (0); - if (!(server_info.wordDict = dict_open (FNAME_WORD_DICT, 40, 0))) - { - logf (LOG_WARN, "dict_open fail: word dict"); - r.errcode = 1; - r.errstring = "dict_open fail: word dict"; - return &r; - } - if (!(server_info.wordIsam = is_open (FNAME_WORD_ISAM, key_compare, 0, - sizeof (struct it_key)))) - { - logf (LOG_WARN, "is_open fail: word isam"); - dict_close (server_info.wordDict); - r.errcode = 1; - r.errstring = "is_open fail: word isam"; - return &r; - } -#endif server_info.odr = odr_createmem (ODR_ENCODE); return &r; } @@ -214,7 +217,7 @@ bend_searchresult *bend_search (void *handle, bend_searchrequest *q, int *fd) r.errstring = 0; r.hits = 0; - register_check (&server_info); + register_lock (&server_info); odr_reset (server_info.odr); server_info.errCode = 0; server_info.errString = NULL; @@ -231,6 +234,7 @@ bend_searchresult *bend_search (void *handle, bend_searchrequest *q, int *fd) default: r.errcode = 107; } + register_unlock (&server_info); return &r; } @@ -321,7 +325,7 @@ bend_fetchresult *bend_fetch (void *handle, bend_fetchrequest *q, int *num) int positions[2]; ZServerSetSysno *records; - register_check (&server_info); + register_lock (&server_info); r.errstring = 0; r.last_in_set = 0; @@ -336,24 +340,28 @@ bend_fetchresult *bend_fetch (void *handle, bend_fetchrequest *q, int *num) { logf (LOG_DEBUG, "resultSetRecordGet, error"); r.errcode = 13; + register_unlock (&server_info); return &r; } if (!records[0].sysno) { r.errcode = 13; logf (LOG_DEBUG, "Out of range. pos=%d", q->number); + register_unlock (&server_info); return &r; } r.errcode = record_fetch (&server_info, records[0].sysno, records[0].score, q->stream, q->format, q->comp, &r.format, &r.record, &r.len); resultSetSysnoDel (&server_info, records, 1); + register_unlock (&server_info); return &r; } bend_deleteresult *bend_delete (void *handle, bend_deleterequest *q, int *num) { - register_check (&server_info); + register_lock (&server_info); + register_unlock (&server_info); return 0; } @@ -362,7 +370,7 @@ bend_scanresult *bend_scan (void *handle, bend_scanrequest *q, int *num) static bend_scanresult r; int status; - register_check (&server_info); + register_lock (&server_info); odr_reset (server_info.odr); server_info.errCode = 0; server_info.errString = 0; @@ -375,6 +383,7 @@ bend_scanresult *bend_scan (void *handle, bend_scanrequest *q, int *num) &r.num_entries, &r.entries, &status); r.errstring = server_info.errString; r.status = status; + register_unlock (&server_info); return &r; } @@ -385,8 +394,8 @@ void bend_close (void *handle) dict_close (server_info.wordDict); is_close (server_info.wordIsam); rec_close (&server_info.records); + register_unlock (&server_info); } - zebraServerUnlock (); return; } diff --git a/index/zserver.h b/index/zserver.h index d234356..9ee5afc 100644 --- a/index/zserver.h +++ b/index/zserver.h @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zserver.h,v $ - * Revision 1.16 1995-12-07 17:38:48 adam + * Revision 1.17 1995-12-08 16:22:57 adam + * Work on update while servers are running. Three lock files introduced. + * The servers reload their registers when necessary, but they don't + * reestablish result sets yet. + * + * Revision 1.16 1995/12/07 17:38:48 adam * Work locking mechanisms for concurrent updates/commit. * * Revision 1.15 1995/11/21 15:29:13 adam @@ -76,6 +81,7 @@ typedef struct ZServerSet_ { typedef struct { int registerState; /* 0 (no commit pages), 1 (use commit pages) */ + time_t registerChange; ZServerSet *sets; Dict wordDict; ISAM wordIsam; @@ -101,3 +107,7 @@ ZServerSetSysno *resultSetSysnoGet (ZServerInfo *zi, const char *name, int num, int *positions); void resultSetSysnoDel (ZServerInfo *zi, ZServerSetSysno *records, int num); void zlog_rpn (Z_RPNQuery *rpn); + +int zebraServerLock (int lockCommit); +void zebraServerUnlock (int commitPhase); +int zebraServerLockGetState (time_t *timep);