From d75b9accf5a28bd5d8ffd70bbb33b3e8e009d079 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Thu, 30 Nov 1995 08:33:09 +0000 Subject: [PATCH] Started work on commit facility. --- bfile/Makefile | 4 +- bfile/bfile.c | 87 ++++++++++-- bfile/cfile.c | 401 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ bfile/cfile.h | 58 ++++++++ bfile/commit.c | 66 +++++++++ include/bfile.h | 16 ++- include/mfile.h | 7 +- 7 files changed, 621 insertions(+), 18 deletions(-) create mode 100644 bfile/cfile.c create mode 100644 bfile/cfile.h create mode 100644 bfile/commit.c diff --git a/bfile/Makefile b/bfile/Makefile index fd07e99..aafb05e 100644 --- a/bfile/Makefile +++ b/bfile/Makefile @@ -1,7 +1,7 @@ # Copyright (C) 1994, Index Data I/S # All rights reserved. # Sebastian Hammer, Adam Dickmeiss -# $Id: Makefile,v 1.14 1995-11-20 11:57:54 adam Exp $ +# $Id: Makefile,v 1.15 1995-11-30 08:33:09 adam Exp $ SHELL=/bin/sh RANLIB=ranlib @@ -16,7 +16,7 @@ TPROG=btest CFLAGS=-g -Wall -pedantic -ansi DEFS=$(INCLUDE) LIB=../lib/bfile.a -PO = bfile.o mfile.o +PO = bfile.o mfile.o cfile.o commit.o CPP=$(CC) -E all: $(LIB) diff --git a/bfile/bfile.c b/bfile/bfile.c index 7b65809..bf73dc3 100644 --- a/bfile/bfile.c +++ b/bfile/bfile.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: bfile.c,v $ - * Revision 1.11 1995-09-04 12:33:21 adam + * Revision 1.12 1995-11-30 08:33:10 adam + * Started work on commit facility. + * + * Revision 1.11 1995/09/04 12:33:21 adam * Various cleanup. YAZ util used instead. * * Revision 1.10 1994/08/25 10:15:54 quinn @@ -44,32 +47,98 @@ #include #include +#include "cfile.h" + +static const char *cache_name = NULL; + +void bf_cache (const char *name) +{ + cache_name = name; +} int bf_close (BFile bf) { - mf_close(bf->mf); - xfree(bf); - return(0); + if (bf->cf) + cf_close (bf->cf); + mf_close (bf->mf); + xfree (bf); + return 0; } BFile bf_open (const char *name, int block_size, int wflag) { BFile tmp = xmalloc(sizeof(BFile_struct)); - if (!(tmp->mf = mf_open(0, name, block_size, wflag))) + if (cache_name) + { + FILE *outf; + int first_time; + + logf (LOG_LOG, "cf,mf_open %s, cache_name=%s", name, cache_name); + tmp->mf = mf_open(0, name, block_size, wflag); + tmp->cf = cf_open(tmp->mf, cache_name, name, block_size, wflag, + &first_time); + + if (first_time) + { + outf = fopen (cache_name, "a"); + fprintf (outf, "%s %d\n", name, block_size); + fclose (outf); + } + } + else + { + tmp->mf = mf_open(0, name, block_size, wflag); + tmp->cf = NULL; + } + if (!tmp->mf) { - logf (LOG_FATAL, "Mfopen failed for %s", name); - return(0); + logf (LOG_FATAL, "mf_open failed for %s", name); + xfree (tmp); + return 0; } return(tmp); } int bf_read (BFile bf, int no, int offset, int num, void *buf) { - return mf_read(bf->mf, no, offset, num, buf); + int r; + + if (bf->cf && (r=cf_read (bf->cf, no, offset, num, buf)) != -1) + return r; + return mf_read (bf->mf, no, offset, num, buf); } int bf_write (BFile bf, int no, int offset, int num, const void *buf) { - return mf_write(bf->mf, no, offset, num, buf); + if (bf->cf) + return cf_write (bf->cf, no, offset, num, buf); + return mf_write (bf->mf, no, offset, num, buf); +} + +void bf_commit (const char *name) +{ + FILE *inf; + int block_size; + char path[256]; + MFile mf; + CFile cf; + int first_time; + + if (!(inf = fopen (name, "r"))) + { + logf (LOG_FATAL|LOG_ERRNO, "cannot open commit %s", name); + exit (1); + } + while (fscanf (inf, "%s %d", path, &block_size) == 1) + { + mf = mf_open(0, path, block_size, 1); + cf = cf_open(mf, name, path, block_size, 0, &first_time); + + cf_commit (cf); + + cf_close (cf); + mf_close (mf); + } + fclose (inf); } diff --git a/bfile/cfile.c b/bfile/cfile.c new file mode 100644 index 0000000..ee11f0f --- /dev/null +++ b/bfile/cfile.c @@ -0,0 +1,401 @@ +/* + * Copyright (C) 1995, Index Data I/S + * All rights reserved. + * Sebastian Hammer, Adam Dickmeiss + * + * $Log: cfile.c,v $ + * Revision 1.1 1995-11-30 08:33:11 adam + * Started work on commit facility. + * + */ + +#include +#include +#include +#include + +#include +#include "cfile.h" + +static int hash_write (CFile cf, const void *buf, size_t bytes) +{ + int r; + + r = write (cf->hash_fd, buf, bytes); + if (r == bytes) + return bytes; + if (r == -1) + logf (LOG_FATAL|LOG_ERRNO, "write in commit hash file"); + else + logf (LOG_FATAL, "write in commit hash file. " + "%d bytes instead of %d bytes", r, bytes); + exit (1); + return 0; +} + +static int hash_read (CFile cf, void *buf, size_t bytes) +{ + int r; + + r = read (cf->hash_fd, buf, bytes); + if (r == bytes) + return bytes; + if (r == -1) + logf (LOG_FATAL|LOG_ERRNO, "read in commit hash file"); + else + logf (LOG_FATAL, "read in commit hash file. " + "%d bytes instead of %d bytes", r, bytes); + abort (); + return 0; +} + +CFile cf_open (MFile mf, const char *cname, const char *fname, + int block_size, int wflag, int *firstp) +{ + char path[256]; + int r, i; + CFile cf = xmalloc (sizeof(*cf)); + int hash_bytes; + + cf->mf = mf; + sprintf (path, "%s.%s.b", cname, fname); + if ((cf->block_fd = + open (path, wflag ? O_RDWR|O_CREAT : O_RDONLY, 0666)) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "Failed to open %s", path); + exit (1); + } + sprintf (path, "%s.%s.h", cname, fname); + if ((cf->hash_fd = + open (path, wflag ? O_RDWR|O_CREAT : O_RDONLY, 0666)) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "Failed to open %s", path); + exit (1); + } + r = read (cf->hash_fd, &cf->head, sizeof(cf->head)); + if (r != sizeof(cf->head)) + { + *firstp = 1; + if (r == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "read head of %s", path); + exit (1); + } + if (r != 0) + { + logf (LOG_FATAL, "illegal head of %s", path); + exit (1); + } + cf->head.block_size = block_size; + cf->head.hash_size = 401; + hash_bytes = cf->head.hash_size * sizeof(int); + cf->head.next_bucket = + (hash_bytes+sizeof(cf->head))/HASH_BSIZE + 2; + cf->head.next_block = 1; + if (wflag) + hash_write (cf, &cf->head, sizeof(cf->head)); + cf->array = xmalloc (hash_bytes); + for (i = 0; ihead.hash_size; i++) + cf->array[i] = 0; + if (wflag) + hash_write (cf, cf->array, hash_bytes); + } + else + { + *firstp = 0; + assert (cf->head.block_size == block_size); + assert (cf->head.hash_size > 2 && cf->head.hash_size < 200000); + hash_bytes = cf->head.hash_size * sizeof(int); + assert (cf->head.next_bucket > 0); + cf->array = xmalloc (hash_bytes); + hash_read (cf, cf->array, hash_bytes); + } + cf->parray = xmalloc (cf->head.hash_size * sizeof(*cf->parray)); + for (i = 0; ihead.hash_size; i++) + cf->parray[i] = NULL; + cf->bucket_lru_front = cf->bucket_lru_back = NULL; + cf->bucket_in_memory = 0; + cf->max_bucket_in_memory = 200; + cf->dirty = 0; + cf->iobuf = xmalloc (cf->head.block_size); + return cf; +} + +static int cf_hash (CFile cf, int no) +{ + return (no>>3) % cf->head.hash_size; +} + +static void release_bucket (CFile cf, struct CFile_hash_bucket *p) +{ + if (p->lru_prev) + p->lru_prev->lru_next = p->lru_next; + else + cf->bucket_lru_back = p->lru_next; + if (p->lru_next) + p->lru_next->lru_prev = p->lru_prev; + else + cf->bucket_lru_front = p->lru_prev; + + *p->h_prev = p->h_next; + if (p->h_next) + p->h_next->h_prev = p->h_prev; + + --(cf->bucket_in_memory); + xfree (p); +} + +static void flush_bucket (CFile cf, int no_to_flush) +{ + int i; + struct CFile_hash_bucket *p; + + for (i = 0; i != no_to_flush; i++) + { + p = cf->bucket_lru_back; + if (!p) + break; + if (p->dirty) + { + if (lseek (cf->hash_fd, p->ph.this_bucket*HASH_BSIZE, SEEK_SET) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "lseek in flush_bucket"); + exit (1); + } + hash_write (cf, &p->ph, HASH_BSIZE); + cf->dirty = 1; + } + release_bucket (cf, p); + } +} + +static struct CFile_hash_bucket *alloc_bucket (CFile cf, int block_no, int hno) +{ + struct CFile_hash_bucket *p, **pp; + + if (cf->bucket_in_memory == cf->max_bucket_in_memory) + flush_bucket (cf, 1); + assert (cf->bucket_in_memory < cf->max_bucket_in_memory); + ++(cf->bucket_in_memory); + p = xmalloc (sizeof(*p)); + + p->lru_next = NULL; + p->lru_prev = cf->bucket_lru_front; + if (cf->bucket_lru_front) + cf->bucket_lru_front->lru_next = p; + else + cf->bucket_lru_back = p; + cf->bucket_lru_front = p; + + pp = cf->parray + hno; + p->h_next = *pp; + p->h_prev = pp; + if (*pp) + (*pp)->h_prev = &p->h_next; + *pp = p; + return p; +} + +static struct CFile_hash_bucket *get_bucket (CFile cf, int block_no, int hno) +{ + struct CFile_hash_bucket *p; + + p = alloc_bucket (cf, block_no, hno); + + if (lseek (cf->hash_fd, block_no * HASH_BSIZE, SEEK_SET) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "lseek in get_bucket"); + exit (1); + } + hash_read (cf, &p->ph, HASH_BSIZE); + assert (p->ph.this_bucket == block_no); + p->dirty = 0; + return p; +} + +static struct CFile_hash_bucket *new_bucket (CFile cf, int *block_no, int hno) +{ + struct CFile_hash_bucket *p; + int i; + + *block_no = cf->head.next_bucket++; + p = alloc_bucket (cf, *block_no, hno); + + for (i = 0; iph.vno[i] = 0; + p->ph.next_bucket = 0; + p->ph.this_bucket = *block_no; + p->dirty = 1; + return p; +} + +int cf_lookup (CFile cf, int no) +{ + int hno = cf_hash (cf, no); + struct CFile_hash_bucket *hb; + int block_no, i; + + logf (LOG_LOG, "cf_lookup pass 1"); + for (hb = cf->parray[hno]; hb; hb = hb->h_next) + { + logf (LOG_LOG, "bucket_no=%d", hb->ph.this_bucket); + for (i = 0; iph.vno[i]; i++) + if (hb->ph.no[i] == no) + return hb->ph.vno[i]; + } + logf (LOG_LOG, "cf_lookup pass 2"); + for (block_no = cf->array[hno]; block_no; block_no = hb->ph.next_bucket) + { + logf (LOG_LOG, "bucket_no=%d", block_no); + for (hb = cf->parray[hno]; hb; hb = hb->h_next) + if (hb->ph.this_bucket == block_no) + continue; + hb = get_bucket (cf, block_no, hno); + for (i = 0; iph.vno[i]; i++) + if (hb->ph.no[i] == no) + return hb->ph.vno[i]; + } + return 0; +} + +int cf_new (CFile cf, int no) +{ + int hno = cf_hash (cf, no); + struct CFile_hash_bucket *hbprev = NULL, *hb = cf->parray[hno]; + int *bucketpp = &cf->array[hno]; + int i; + int vno = (cf->head.next_block)++; + + for (hb = cf->parray[hno]; hb; hb = hb->h_next) + if (!hb->ph.vno[HASH_BUCKET-1]) + for (i = 0; iph.vno[i]) + { + hb->ph.no[i] = no; + hb->ph.vno[i] = vno; + hb->dirty = 1; + return vno; + } + + while (*bucketpp) + { + for (hb = cf->parray[hno]; hb; hb = hb->h_next) + if (hb->ph.this_bucket == *bucketpp) + { + bucketpp = &hb->ph.next_bucket; + hbprev = hb; + continue; + } + hb = get_bucket (cf, *bucketpp, hno); + assert (hb); + for (i = 0; iph.vno[i]) + { + hb->ph.no[i] = no; + hb->ph.vno[i] = vno; + hb->dirty = 1; + return vno; + } + bucketpp = &hb->ph.next_bucket; + hbprev = hb; + } + if (hbprev) + hbprev->dirty = 1; + hb = new_bucket (cf, bucketpp, hno); + hb->ph.no[0] = no; + hb->ph.vno[0] = vno; + return vno; +} + +int cf_read (CFile cf, int no, int offset, int num, void *buf) +{ + int block, r; + + assert (cf); + logf (LOG_LOG, "cf_read no=%d, offset=%d, num=%d", no, offset, num); + if (!(block = cf_lookup (cf, no))) + return -1; + if (lseek (cf->block_fd, cf->head.block_size * block + offset, + SEEK_SET) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "cf_read, lseek no=%d, block=%d", + no, block); + exit (1); + } + r = read (cf->block_fd, buf, num ? num : cf->head.block_size); + if (r != cf->head.block_size) + { + logf (LOG_FATAL|LOG_ERRNO, "cf_read, read no=%d, block=%d", + no, block); + exit (1); + } + return 1; +} + +int cf_write (CFile cf, int no, int offset, int num, const void *buf) +{ + int block, r; + + assert (cf); + + logf (LOG_LOG, "cf_write no=%d, offset=%d, num=%d", no, offset, num); + if (!(block = cf_lookup (cf, no))) + { + block = cf_new (cf, no); + if (offset || num) + { + mf_read (cf->mf, no, 0, 0, cf->iobuf); + memcpy (cf->iobuf + offset, buf, num); + buf = cf->iobuf; + offset = 0; + num = 0; + } + } + if (lseek (cf->block_fd, cf->head.block_size * block + offset, + SEEK_SET) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "cf_write, lseek no=%d, block=%d", + no, block); + exit (1); + } + r = write (cf->block_fd, buf, num ? num : cf->head.block_size); + if (r != cf->head.block_size) + { + logf (LOG_FATAL|LOG_ERRNO, "cf_write, read no=%d, block=%d", + no, block); + exit (1); + } + return 0; +} + +int cf_close (CFile cf) +{ + flush_bucket (cf, -1); + if (cf->dirty) + { + int hash_bytes = cf->head.hash_size * sizeof(int); + if (lseek (cf->hash_fd, 0L, SEEK_SET) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "seek in hash fd"); + exit (1); + } + hash_write (cf, &cf->head, sizeof(cf->head)); + hash_write (cf, cf->array, hash_bytes); + } + if (close (cf->hash_fd) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "close hash fd"); + exit (1); + } + if (close (cf->block_fd) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "close block fd"); + exit (1); + } + xfree (cf->array); + xfree (cf->parray); + xfree (cf->iobuf); + xfree (cf); + return 0; +} + diff --git a/bfile/cfile.h b/bfile/cfile.h new file mode 100644 index 0000000..5080151 --- /dev/null +++ b/bfile/cfile.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 1995, Index Data I/S + * All rights reserved. + * Sebastian Hammer, Adam Dickmeiss + * + * $Log: cfile.h,v $ + * Revision 1.1 1995-11-30 08:33:12 adam + * Started work on commit facility. + * + */ + +#ifndef CFILE_H +#define CFILE_H + +#define HASH_BUCKET 63 + +struct CFile_hash_bucket { + struct CFile_ph_bucket { + int no[HASH_BUCKET]; + int vno[HASH_BUCKET]; + int this_bucket; + int next_bucket; + } ph; + int dirty; + struct CFile_hash_bucket *h_next, **h_prev; + struct CFile_hash_bucket *lru_next, *lru_prev; +}; + +#define HASH_BSIZE sizeof(struct CFile_ph_bucket) + +typedef struct CFile_struct +{ + struct CFile_head { + int hash_size; + int next_bucket; + int next_block; + int block_size; + } head; + int block_fd; + int hash_fd; + int *array; + struct CFile_hash_bucket **parray; + struct CFile_hash_bucket *bucket_lru_front, *bucket_lru_back; + int dirty; + int bucket_in_memory; + int max_bucket_in_memory; + char *iobuf; + MFile mf; +} *CFile; + +int cf_close (CFile cf); +CFile cf_open (MFile mf, const char *cname, const char *fname, + int block_size, int wflag, int *firstp); +int cf_read (CFile cf, int no, int offset, int num, void *buf); +int cf_write (CFile cf, int no, int offset, int num, const void *buf); +void cf_commit (CFile cf); + +#endif diff --git a/bfile/commit.c b/bfile/commit.c new file mode 100644 index 0000000..216825c --- /dev/null +++ b/bfile/commit.c @@ -0,0 +1,66 @@ +/* + * Copyright (C) 1995, Index Data I/S + * All rights reserved. + * Sebastian Hammer, Adam Dickmeiss + * + * $Log: commit.c,v $ + * Revision 1.1 1995-11-30 08:33:13 adam + * Started work on commit facility. + * + */ + +#include +#include +#include +#include + +#include +#include "cfile.h" + +void cf_commit (CFile cf) +{ + int i, r, bucket_no; + int hash_bytes; + struct CFile_ph_bucket *p; + + if (cf->bucket_in_memory) + { + logf (LOG_FATAL, "Cannot commit potential dirty cache"); + exit (1); + } + p = xmalloc (sizeof(*p)); + hash_bytes = cf->head.hash_size * sizeof(int); + bucket_no = (hash_bytes+sizeof(cf->head))/HASH_BSIZE + 2; + if (lseek (cf->hash_fd, bucket_no * HASH_BSIZE, SEEK_SET) < 0) + { + logf (LOG_FATAL|LOG_ERRNO, "seek commit"); + exit (1); + } + for (; bucket_no < cf->head.next_bucket; bucket_no++) + { + r = read (cf->hash_fd, p, HASH_BSIZE); + if (r != HASH_BSIZE) + { + logf (LOG_FATAL, "read commit hash"); + exit (1); + } + for (i = 0; ivno[i]; i++) + { + if (lseek (cf->block_fd, p->vno[i]*cf->head.block_size, + SEEK_SET) < 0) + { + logf (LOG_FATAL, "lseek commit block"); + exit (1); + } + r = read (cf->block_fd, cf->iobuf, cf->head.block_size); + if (r != cf->head.block_size) + { + logf (LOG_FATAL, "read commit block"); + exit (1); + } + mf_write (cf->mf, p->no[i], 0, 0, cf->iobuf); + } + } + xfree (p); +} + diff --git a/include/bfile.h b/include/bfile.h index 2290fa8..fa06467 100644 --- a/include/bfile.h +++ b/include/bfile.h @@ -1,12 +1,15 @@ /* - * Copyright (C) 1994, Index Data I/S + * Copyright (C) 1994-1995, Index Data I/S * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * - $Log: bfile.h,v $ - Revision 1.7 1995-09-04 12:33:35 adam - Various cleanup. YAZ util used instead. - + * $Log: bfile.h,v $ + * Revision 1.8 1995-11-30 08:33:29 adam + * Started work on commit facility. + * + * Revision 1.7 1995/09/04 12:33:35 adam + * Various cleanup. YAZ util used instead. + * * Revision 1.6 1994/09/14 13:10:35 quinn * Small changes * @@ -32,11 +35,14 @@ typedef struct BFile_struct { MFile mf; + struct CFile_struct *cf; } *BFile, BFile_struct; int bf_close (BFile); BFile bf_open (const char *name, int block_size, int wflag); int bf_read (BFile bf, int no, int offset, int num, void *buf); int bf_write (BFile bf, int no, int offset, int num, const void *buf); +void bf_cache (const char *name); +void bf_commit (const char *name); #endif diff --git a/include/mfile.h b/include/mfile.h index c99fc50..3bd0591 100644 --- a/include/mfile.h +++ b/include/mfile.h @@ -1,10 +1,13 @@ /* - * Copyright (C) 1994, Index Data I/S + * Copyright (C) 1994-1995, Index Data I/S * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: mfile.h,v $ - * Revision 1.3 1995-09-04 12:33:35 adam + * Revision 1.4 1995-11-30 08:33:30 adam + * Started work on commit facility. + * + * Revision 1.3 1995/09/04 12:33:35 adam * Various cleanup. YAZ util used instead. * * Revision 1.2 1994/09/14 13:10:36 quinn -- 1.7.10.4