From: Adam Dickmeiss <adam@indexdata.dk>
Date: Thu, 7 Dec 1995 17:38:45 +0000 (+0000)
Subject: Work locking mechanisms for concurrent updates/commit.
X-Git-Tag: ZEBRA.1.0~604
X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=commitdiff_plain;h=4e2fc80e64f3d7895a871ea5b08aa80045a59f2c

Work locking mechanisms for concurrent updates/commit.
---

diff --git a/index/Makefile b/index/Makefile
index 078d52c..89286f6 100644
--- a/index/Makefile
+++ b/index/Makefile
@@ -1,7 +1,7 @@
 # Copyright (C) 1995, Index Data I/S 
 # All rights reserved.
 # Sebastian Hammer, Adam Dickmeiss
-# $Id: Makefile,v 1.28 1995-12-06 12:41:20 adam Exp $
+# $Id: Makefile,v 1.29 1995-12-07 17:38:45 adam Exp $
 
 SHELL=/bin/sh
 RANLIB=ranlib
@@ -19,10 +19,11 @@ TPROG2=kdump
 TPROG3=zebrasrv
 DEFS=$(INCLUDE)
 O1 = main.o dir.o dirs.o trav.o extract.o kinput.o kcompare.o \
- symtab.o text.o recctrl.o structrec.o recindex.o regxread.o recstat.o
+ symtab.o text.o recctrl.o structrec.o recindex.o regxread.o recstat.o \
+ lockutil.o lockidx.o
 O2 = kdump.o
 O3 = zserver.o kcompare.o zrpn.o zsets.o text.o recctrl.o structrec.o  \
-	attribute.o recindex.o zlogs.o regxread.o
+	attribute.o recindex.o zlogs.o regxread.o lockutil.o locksrv.o
 CPP=$(CC) -E
 
 all: $(TPROG1) $(TPROG2) $(TPROG3)
diff --git a/index/extract.c b/index/extract.c
index 4426e4a..8b189ca 100644
--- a/index/extract.c
+++ b/index/extract.c
@@ -4,7 +4,10 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: extract.c,v $
- * Revision 1.41  1995-12-06 16:06:42  adam
+ * Revision 1.42  1995-12-07 17:38:46  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.41  1995/12/06  16:06:42  adam
  * Better diagnostics. Work on 'real' dictionary deletion.
  *
  * Revision 1.40  1995/12/05  16:57:40  adam
@@ -170,8 +173,6 @@ static int records_inserted = 0;
 static int records_updated = 0;
 static int records_deleted = 0;
 
-#define MATCH_DICT "match"
-
 void key_open (int mem)
 {
     if (mem < 50000)
@@ -183,9 +184,9 @@ void key_open (int mem)
     key_buf_used = 0;
     key_file_no = 0;
 
-    if (!(matchDict = dict_open (MATCH_DICT, 20, 1)))
+    if (!(matchDict = dict_open (GMATCH_DICT, 50, 1)))
     {
-        logf (LOG_FATAL, "dict_open fail of %s", MATCH_DICT);
+        logf (LOG_FATAL, "dict_open fail of %s", GMATCH_DICT);
         exit (1);
     }
     assert (!records);
@@ -797,11 +798,10 @@ static int recordExtract (SYSNO *sysno, const char *fname,
             }
             else
             {
-                SYSNO sysnoz = 0;
                 logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname);
                 records_deleted++;
                 if (matchStr)
-                    dict_insert (matchDict, matchStr, sizeof(sysnoz), &sysnoz);
+                    dict_delete (matchDict, matchStr);
                 rec_del (records, &rec);
             }
             return 1;
diff --git a/index/index.h b/index/index.h
index d40a588..620ffcf 100644
--- a/index/index.h
+++ b/index/index.h
@@ -4,7 +4,10 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: index.h,v $
- * Revision 1.31  1995-12-06 12:41:22  adam
+ * Revision 1.32  1995-12-07 17:38:46  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.31  1995/12/06  12:41:22  adam
  * New command 'stat' for the index program.
  * Filenames can be read from stdin by specifying '-'.
  * Bug fix/enhancement of the transformation from terms to regular
@@ -183,6 +186,9 @@ int merge_sort (char **buf, int from, int to);
 #define FNAME_WORD_ISAM "wordisam"
 #define FNAME_CONFIG "zebra.cfg"
 
+#define GMATCH_DICT "gmatch"
+#define FMATCH_DICT "fmatch"
+
 struct strtab *strtab_mk (void);
 int strtab_src (struct strtab *t, const char *name, void ***infop);
 void strtab_del (struct strtab *t,
@@ -196,3 +202,16 @@ int fileExtract (SYSNO *sysno, const char *fname,
                  const struct recordGroup *rGroup, int deleteFlag);
 
 void rec_prstat (void);
+
+void zebraLockPrefix (char *pathPrefix);
+
+int zebraServerLock (void);
+void zebraServerUnlock (void);
+
+void zebraIndexLockMsg (const char *str);
+void zebraIndexUnlock (int rw);
+void zebraIndexLock (int rw);
+void zebraIndexLockCommit (void);
+int zebraServerLockGetState (void);
+
+#define FNAME_MAIN_LOCK "zebra.lock"
diff --git a/index/lockidx.c b/index/lockidx.c
new file mode 100644
index 0000000..ed7c7f8
--- /dev/null
+++ b/index/lockidx.c
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 1994-1995, Index Data I/S 
+ * All rights reserved.
+ * Sebastian Hammer, Adam Dickmeiss
+ *
+ * $Log: lockidx.c,v $
+ * Revision 1.1  1995-12-07 17:38:47  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ */
+#include <stdio.h>
+#include <assert.h>
+#include <unistd.h>
+#include <sys/file.h>
+#include <fcntl.h>
+#include <string.h>
+#include <errno.h>
+
+#include <alexutil.h>
+#include "index.h"
+
+static int lock_fd = -1;
+
+void zebraIndexLockMsg (const char *str)
+{
+    int l, r;
+
+    assert (lock_fd != -1);
+    lseek (lock_fd, 0L, SEEK_SET);
+    l = strlen(str);
+    r = write (lock_fd, str, l);
+    if (r != l)
+    {
+        logf (LOG_FATAL|LOG_ERRNO, "write lock file");
+        exit (1);
+    }
+}
+
+void zebraIndexUnlock (int rw)
+{
+    char path[1024];
+    char pathPrefix[1024];
+
+    if (lock_fd == -1)
+        return;
+
+    zebraLockPrefix (pathPrefix);
+    sprintf (path, "%s%s", pathPrefix, FNAME_MAIN_LOCK);
+    unlink (path);
+}
+
+void zebraIndexLock (int rw)
+{
+    char path[1024];
+    char pathPrefix[1024];
+    char buf[256];
+    int r;
+
+    zebraLockPrefix (pathPrefix);
+    sprintf (path, "%s%s", pathPrefix, FNAME_MAIN_LOCK);
+    if (rw)
+    {
+        while (1)
+        {
+            lock_fd = open (path, O_CREAT|O_RDWR|O_EXCL|O_SYNC, 0666);
+            if (lock_fd == -1)
+            {
+                lock_fd = open (path, O_RDONLY);
+                assert (lock_fd != -1);
+                if (flock (lock_fd, LOCK_EX|LOCK_NB) == -1)
+                {
+                    if (errno == EWOULDBLOCK)
+                    {
+                        logf (LOG_LOG, "Waiting for other index process");
+                        flock (lock_fd, LOCK_EX);
+                        continue;
+                    }
+                    else
+                    {
+                        logf (LOG_FATAL|LOG_ERRNO, "flock %s", path);
+                        exit (1);
+                    }
+                }
+                else
+                {
+                    logf (LOG_WARN, "Unlocked %s", path);
+                    r = read (lock_fd, buf, 256);
+                    if (r == 0)
+                    {
+                        logf (LOG_WARN, "Zero length %s", path);
+                        unlink (path);
+                        close (lock_fd);
+                        continue;
+                    }
+                    else if (r == -1)
+                    {
+                        logf (LOG_FATAL|LOG_ERRNO, "read  %s", path);
+                        exit (1);
+                    }
+                    if (*buf == 'r')
+                    {
+                        logf (LOG_WARN, "Previous transaction didn't"
+                              " reach commit");
+                        unlink (path);
+                        close (lock_fd);
+                        continue;
+                    }
+                    else if (*buf == 'w')
+                    {
+                        logf (LOG_WARN, "Your index may be inconsistent");
+                        exit (1);
+                    }
+                    else if (*buf == 'c')
+                    {
+                        logf (LOG_FATAL, "Previous transaction didn't"
+                              " finish commit. Restore now!");
+                        exit (1);
+                    }
+                    else 
+                    {
+                        logf (LOG_FATAL, "Unknown id 0x%02x in %s", *buf,
+                              path);
+                        exit (1);
+                    }
+                }
+            }
+            else
+                break;
+        }
+        flock (lock_fd, LOCK_EX);
+    }
+}
+
+void zebraIndexLockCommit (void)
+{
+
+}
+
diff --git a/index/locksrv.c b/index/locksrv.c
new file mode 100644
index 0000000..517e680
--- /dev/null
+++ b/index/locksrv.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 1994-1995, Index Data I/S 
+ * All rights reserved.
+ * Sebastian Hammer, Adam Dickmeiss
+ *
+ * $Log: locksrv.c,v $
+ * Revision 1.1  1995-12-07 17:38:47  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ */
+#include <stdio.h>
+#include <assert.h>
+#include <unistd.h>
+#include <sys/file.h>
+#include <fcntl.h>
+#include <string.h>
+#include <errno.h>
+
+#include <alexutil.h>
+#include "index.h"
+
+static int server_lock_fd = -1;
+
+int zebraServerLock (void)
+{
+    char pathPrefix[1024];
+    char path[1024];
+    
+    zebraLockPrefix (pathPrefix);
+
+    assert (server_lock_fd == -1);
+    sprintf (path, "%szebrasrv.%ld", pathPrefix, (long) getpid());
+    if ((server_lock_fd = open (path, O_CREAT|O_RDWR|O_SYNC|O_EXCL, 0666))
+        == -1)
+    {
+        logf (LOG_WARN|LOG_ERRNO, "remove stale %s", path);
+        unlink (path);
+        if ((server_lock_fd = open (path, O_CREAT|O_RDWR|O_SYNC|O_EXCL, 0666))
+            == -1)
+        {
+            logf (LOG_FATAL|LOG_ERRNO, "create %s", path);
+            return -1;
+        }
+    }
+    flock (server_lock_fd, LOCK_EX);
+
+    return 0;
+}
+
+int zebraServerLockGetState (void)
+{
+    char pathPrefix[1024];
+    char path[1024];
+    char buf[256];
+    int fd;
+    
+    zebraLockPrefix (pathPrefix);
+
+    sprintf (path, "%s%s", pathPrefix, FNAME_MAIN_LOCK);
+    fd = open (path, O_RDONLY);
+    if (fd == -1)
+        return 0;
+    if (read (fd, buf, 2) == 0)
+        return 0;
+    close (fd);
+    return *buf;
+}
+
+void zebraServerLockMsg (const char *str)
+{
+    int l, r;
+
+    assert (server_lock_fd != -1);
+    lseek (server_lock_fd, 0L, SEEK_SET);
+    l = strlen(str);
+    r = write (server_lock_fd, str, l);
+    if (r != l)
+    {
+        logf (LOG_FATAL|LOG_ERRNO, "write server lock file");
+        exit (1);
+    }
+}
+
+void zebraServerUnlock (void)
+{
+    char pathPrefix[1024];
+    char path[1024];
+    
+    assert (server_lock_fd != -1);
+    zebraLockPrefix (pathPrefix);
+    flock (server_lock_fd, LOCK_UN);
+    sprintf (path, "%szebrasrv.%ld", pathPrefix, (long) getpid());
+    unlink (path);
+}
+
diff --git a/index/lockutil.c b/index/lockutil.c
new file mode 100644
index 0000000..acd6541
--- /dev/null
+++ b/index/lockutil.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 1994-1995, Index Data I/S 
+ * All rights reserved.
+ * Sebastian Hammer, Adam Dickmeiss
+ *
+ * $Log: lockutil.c,v $
+ * Revision 1.1  1995-12-07 17:38:47  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ */
+#include <stdio.h>
+#include <assert.h>
+#include <unistd.h>
+#include <sys/file.h>
+#include <fcntl.h>
+#include <string.h>
+#include <errno.h>
+
+#include <alexutil.h>
+#include "index.h"
+
+static char *lockDir = NULL;
+
+void zebraLockPrefix (char *pathPrefix)
+{
+    if (!lockDir)
+        lockDir = res_get_def (common_resource, "lockDir", "");
+    assert (lockDir);
+    
+    strcpy (pathPrefix, lockDir);
+    if (*pathPrefix && pathPrefix[strlen(pathPrefix)-1] != '/')
+        strcat (pathPrefix, "/");
+}
diff --git a/index/main.c b/index/main.c
index 574c212..5375e09 100644
--- a/index/main.c
+++ b/index/main.c
@@ -4,7 +4,10 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: main.c,v $
- * Revision 1.26  1995-12-06 12:41:23  adam
+ * Revision 1.27  1995-12-07 17:38:47  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.26  1995/12/06  12:41:23  adam
  * New command 'stat' for the index program.
  * Filenames can be read from stdin by specifying '-'.
  * Bug fix/enhancement of the transformation from terms to regular
@@ -162,8 +165,15 @@ int main (int argc, char **argv)
                     }
                     data1_tabpath = res_get (common_resource, "profilePath");
                     rval = res_get (common_resource, "commitEnable");
+
+                    zebraIndexLock (1);
                     if (rval && atoi(rval))
+                    {
+                        zebraIndexLockMsg ("r");
                         bf_cache ();
+                    }
+                    else
+                        zebraIndexLockMsg ("w");
                 }
                 if (!strcmp (arg, "update"))
                     cmd = 'u';
@@ -172,6 +182,7 @@ int main (int argc, char **argv)
                 else if (!strcmp (arg, "commit"))
                 {
                     logf (LOG_LOG, "Commit");
+                    zebraIndexLockMsg ("c");
                     bf_commit ();
                 }
                 else if (!strcmp (arg, "stat") || !strcmp (arg, "status"))
@@ -237,6 +248,7 @@ int main (int argc, char **argv)
             exit (1);
         }
     }
+    zebraIndexUnlock (1);
     exit (0);
 }
 
diff --git a/index/recindex.c b/index/recindex.c
index 893fa05..09cc708 100644
--- a/index/recindex.c
+++ b/index/recindex.c
@@ -4,7 +4,10 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: recindex.c,v $
- * Revision 1.11  1995-12-06 13:58:26  adam
+ * Revision 1.12  1995-12-07 17:38:47  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.11  1995/12/06  13:58:26  adam
  * Improved flushing of records - all flushes except the last one
  * don't write the last accessed. Also flush takes place if record
  * info occupy more than about 256k.
@@ -499,6 +502,7 @@ void rec_del (Records p, Record *recpp)
 {
     Record *recp;
 
+    (p->head.no_records)--;
     if ((recp = rec_cache_lookup (p, (*recpp)->sysno, recordFlagDelete)))
     {
         rec_rm (recp);
diff --git a/index/trav.c b/index/trav.c
index c3f6aca..78746c4 100644
--- a/index/trav.c
+++ b/index/trav.c
@@ -4,7 +4,10 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: trav.c,v $
- * Revision 1.14  1995-12-06 12:41:26  adam
+ * Revision 1.15  1995-12-07 17:38:48  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.14  1995/12/06  12:41:26  adam
  * New command 'stat' for the index program.
  * Filenames can be read from stdin by specifying '-'.
  * Bug fix/enhancement of the transformation from terms to regular
@@ -305,8 +308,11 @@ void repositoryUpdate (struct recordGroup *rGroup)
         Dict dict;
         struct dirs_info *di;
 
-        dict = dict_open ("repdict", 40, 1);
-
+        if (!(dict = dict_open (FMATCH_DICT, 50, 1)))
+        {
+            logf (LOG_FATAL, "dict_open fail of %s", FMATCH_DICT);
+            exit (1);
+        }
         assert (rGroup->path);
         di = dirs_open (dict, rGroup->path);
         strcpy (src, "");
diff --git a/index/zserver.c b/index/zserver.c
index 7bd26e0..c718b70 100644
--- a/index/zserver.c
+++ b/index/zserver.c
@@ -4,7 +4,10 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: zserver.c,v $
- * Revision 1.29  1995-12-04 14:22:32  adam
+ * Revision 1.30  1995-12-07 17:38:48  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.29  1995/12/04  14:22:32  adam
  * Extra arg to recType_byName.
  * Started work on new regular expression parsed input to
  * structured records.
@@ -116,6 +119,38 @@
 
 ZServerInfo server_info;
 
+static int register_check (ZServerInfo *zi)
+{
+    int state = zebraServerLockGetState();
+
+    switch (state)
+    {
+    case 'c':
+        state = 1;
+        break;
+    default:
+        state = 0;
+    }
+    if (zi->registerState == state)
+        return 0;
+
+    zi->registerState = state;
+    if (server_info.records)
+    {
+        dict_close (server_info.wordDict);
+        is_close (server_info.wordIsam);
+        rec_close (&server_info.records);
+    }
+    /* enable commit if state is 1 */
+    server_info.records = rec_open (0);
+    if (!(server_info.wordDict = dict_open (FNAME_WORD_DICT, 40, 0)))
+        return -1;
+    if (!(server_info.wordIsam = is_open (FNAME_WORD_ISAM, key_compare, 0,
+                                          sizeof (struct it_key))))
+        return -1;
+    return 0;
+}
+
 bend_initresult *bend_init (bend_initrequest *q)
 {
     static bend_initresult r;
@@ -139,10 +174,16 @@ bend_initresult *bend_init (bend_initrequest *q)
             exit (1);
         }
     }
+    zebraServerLock ();
 
     data1_tabpath = res_get(common_resource, "profilePath");
     server_info.sets = NULL;
-
+    server_info.registerState = -1;  /* trigger open of registers! */
+#if 1
+    server_info.records = NULL;
+    server_info.wordDict = NULL;
+    server_info.wordIsam = NULL;
+#else
     server_info.records = rec_open (0);
     if (!(server_info.wordDict = dict_open (FNAME_WORD_DICT, 40, 0)))
     {
@@ -160,6 +201,7 @@ bend_initresult *bend_init (bend_initrequest *q)
         r.errstring = "is_open fail: word isam";
         return &r;
     }
+#endif
     server_info.odr = odr_createmem (ODR_ENCODE);
     return &r;
 }
@@ -172,7 +214,7 @@ bend_searchresult *bend_search (void *handle, bend_searchrequest *q, int *fd)
     r.errstring = 0;
     r.hits = 0;
 
-
+    register_check (&server_info);
     odr_reset (server_info.odr);
     server_info.errCode = 0;
     server_info.errString = NULL;
@@ -279,6 +321,8 @@ bend_fetchresult *bend_fetch (void *handle, bend_fetchrequest *q, int *num)
     int positions[2];
     ZServerSetSysno *records;
 
+    register_check (&server_info);
+
     r.errstring = 0;
     r.last_in_set = 0;
     r.basename = "base";
@@ -309,6 +353,7 @@ bend_fetchresult *bend_fetch (void *handle, bend_fetchrequest *q, int *num)
 
 bend_deleteresult *bend_delete (void *handle, bend_deleterequest *q, int *num)
 {
+    register_check (&server_info);
     return 0;
 }
 
@@ -317,6 +362,7 @@ bend_scanresult *bend_scan (void *handle, bend_scanrequest *q, int *num)
     static bend_scanresult r;
     int status;
 
+    register_check (&server_info);
     odr_reset (server_info.odr);
     server_info.errCode = 0;
     server_info.errString = 0;
@@ -334,9 +380,13 @@ bend_scanresult *bend_scan (void *handle, bend_scanrequest *q, int *num)
 
 void bend_close (void *handle)
 {
-    dict_close (server_info.wordDict);
-    is_close (server_info.wordIsam);
-    rec_close (&server_info.records);
+    if (server_info.records)
+    {
+        dict_close (server_info.wordDict);
+        is_close (server_info.wordIsam);
+        rec_close (&server_info.records);
+    }
+    zebraServerUnlock ();
     return;
 }
 
diff --git a/index/zserver.h b/index/zserver.h
index 6bcdf83..d234356 100644
--- a/index/zserver.h
+++ b/index/zserver.h
@@ -4,7 +4,10 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: zserver.h,v $
- * Revision 1.15  1995-11-21 15:29:13  adam
+ * Revision 1.16  1995-12-07 17:38:48  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.15  1995/11/21  15:29:13  adam
  * Config file 'base' read by default by both indexer and server.
  *
  * Revision 1.14  1995/11/16  17:00:57  adam
@@ -72,6 +75,7 @@ typedef struct ZServerSet_ {
 } ZServerSet;
    
 typedef struct {
+    int registerState; /* 0 (no commit pages), 1 (use commit pages) */
     ZServerSet *sets;
     Dict wordDict;
     ISAM wordIsam;