1 /* $Id: mfile.c,v 1.70 2006-11-08 22:08:27 adam Exp $
2 Copyright (C) 1995-2006
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 #include <sys/types.h>
38 #include <zebra-lock.h>
39 #include <idzebra/util.h>
40 #include <yaz/yaz-util.h>
43 static int scan_areadef(MFile_area ma, const char *ad, const char *base)
46 * If no definition is given, use current directory, unlimited.
48 char dirname[FILENAME_MAX+1];
49 mf_dir **dp = &ma->dirs, *dir = *dp;
56 int i = 0, fact = 1, multi;
59 while (*ad == ' ' || *ad == '\t')
63 if (!yaz_is_abspath(ad) && base)
65 strcpy(dirname, base);
71 if (*ad == ':' && strchr("+-0123456789", ad[1]))
80 yaz_log(YLOG_WARN, "Missing colon after path: %s", ad0);
85 yaz_log(YLOG_WARN, "Empty path: %s", ad0);
88 while (*ad == ' ' || *ad == '\t')
98 if (*ad < '0' || *ad > '9')
100 yaz_log(YLOG_FATAL, "Missing size after path: %s", ad0);
104 while (*ad >= '0' && *ad <= '9')
105 size = size*10 + (*ad++ - '0');
108 case 'B': case 'b': multi = 1; break;
109 case 'K': case 'k': multi = 1024; break;
110 case 'M': case 'm': multi = 1048576; break;
111 case 'G': case 'g': multi = 1073741824; break;
113 yaz_log(YLOG_FATAL, "Missing unit: %s", ad0);
116 yaz_log(YLOG_FATAL, "Illegal unit: %c in %s", *ad, ad0);
120 *dp = dir = (mf_dir *) xmalloc(sizeof(mf_dir));
122 strcpy(dir->name, dirname);
123 dir->max_bytes = dir->avail_bytes = fact * size * multi;
129 /** \brief position within metafile (perform seek)
130 \param mf metafile handle
131 \param pos block position
132 \param offset offset within block
135 \retval -2 OK, but file does not created (read-only)
138 static zint file_position(MFile mf, zint pos, int offset)
141 int c = mf->cur_file;
143 if ((c > 0 && pos <= mf->files[c-1].top) ||
144 (c < mf->no_files -1 && pos > mf->files[c].top))
147 while (c + 1 < mf->no_files && mf->files[c].top < pos)
149 off += mf->files[c].blocks;
152 assert(c < mf->no_files);
155 off = c ? (mf->files[c-1].top + 1) : 0;
156 if (mf->files[c].fd < 0)
158 if ((mf->files[c].fd = open(mf->files[c].path,
160 (O_BINARY|O_RDWR|O_CREAT) :
161 (O_BINARY|O_RDONLY), 0666)) < 0)
163 if (!mf->wr && errno == ENOENT && off == 0)
165 /* we can't open it for reading. But not really an error */
168 yaz_log(YLOG_WARN|YLOG_ERRNO, "Failed to open %s", mf->files[c].path);
173 if (mfile_seek(mf->files[c].fd, ps *(mfile_off_t) mf->blocksize + offset,
176 yaz_log(YLOG_WARN|YLOG_ERRNO, "Failed to seek in %s", mf->files[c].path);
177 yaz_log(YLOG_WARN, "pos=" ZINT_FORMAT " off=" ZINT_FORMAT " blocksize=%d offset=%d",
178 pos, off, mf->blocksize, offset);
185 static int cmp_part_file(const void *p1, const void *p2)
187 zint d = ((part_file *)p1)->number - ((part_file *)p2)->number;
195 /** \brief creates a metafile area
196 \param name of area (does not show up on disk - purely for notation)
197 \param spec area specification (e.g. "/a:1G dir /b:2000M"
198 \param base base directory (NULL for no base)
199 \returns metafile area handle or NULL if error occurs
201 MFile_area mf_init(const char *name, const char *spec, const char *base)
203 MFile_area ma = (MFile_area) xmalloc(sizeof(*ma));
206 part_file *part_f = 0;
210 char metaname[FILENAME_MAX+1], tmpnam[FILENAME_MAX+1];
212 yaz_log(YLOG_DEBUG, "mf_init(%s)", name);
213 strcpy(ma->name, name);
216 if (scan_areadef(ma, spec, base) < 0)
218 yaz_log(YLOG_WARN, "Failed to access description of '%s'", name);
221 /* look at each directory */
222 for (dirp = ma->dirs; dirp; dirp = dirp->next)
224 if (!(dd = opendir(dirp->name)))
226 yaz_log(YLOG_WARN|YLOG_ERRNO, "Failed to open directory %s",
230 /* look at each file */
231 while ((dent = readdir(dd)))
233 int len = strlen(dent->d_name);
234 const char *cp = strrchr(dent->d_name, '-');
235 if (strchr(".-", *dent->d_name))
237 if (len < 5 || !cp || strcmp(dent->d_name + len - 3, ".mf"))
240 memcpy(metaname, dent->d_name, cp - dent->d_name);
241 metaname[ cp - dent->d_name] = '\0';
243 for (meta_f = ma->mfiles; meta_f; meta_f = meta_f->next)
246 if (!strcmp(meta_f->name, metaname))
248 part_f = &meta_f->files[meta_f->no_files++];
255 meta_f = (meta_file *) xmalloc(sizeof(*meta_f));
256 zebra_mutex_init(&meta_f->mutex);
258 meta_f->next = ma->mfiles;
260 meta_f->cur_file = -1;
262 strcpy(meta_f->name, metaname);
263 part_f = &meta_f->files[0];
264 meta_f->no_files = 1;
266 part_f->number = number;
269 sprintf(tmpnam, "%s/%s", dirp->name, dent->d_name);
270 part_f->path = xstrdup(tmpnam);
272 if ((fd = open(part_f->path, O_BINARY|O_RDONLY)) < 0)
274 yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to access %s",
278 if ((part_f->bytes = mfile_seek(fd, 0, SEEK_END)) < 0)
280 yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to seek in %s",
288 if (dirp->max_bytes >= 0)
289 dirp->avail_bytes -= part_f->bytes;
293 for (meta_f = ma->mfiles; meta_f; meta_f = meta_f->next)
295 yaz_log(YLOG_DEBUG, "mf_init: %s consists of %d part(s)", meta_f->name,
297 qsort(meta_f->files, meta_f->no_files, sizeof(part_file),
303 /** \brief destroys metafile area handle
304 \param ma metafile area handle
306 void mf_destroy(MFile_area ma)
323 /** \brief reset all files in a metafile area (optionally delete them as well)
324 \param ma metafile area
325 \param unlink_flag if unlink_flag=1 all files are removed from FS
327 void mf_reset(MFile_area ma, int unlink_flag)
337 meta_file *m = meta_f;
339 meta_f = meta_f->next;
342 for (i = 0; i<m->no_files; i++)
345 unlink(m->files[i].path);
346 xfree(m->files[i].path);
348 zebra_mutex_destroy(&m->mutex);
354 /** \brief opens metafile
355 \param ma metafile area handle
356 \param name pseudo filename (name*.mf)
357 \param block_size block size for this file
358 \param wflag write flag, 0=read, 1=write&read
359 \returns metafile handle, or NULL for error (could not be opened)
361 MFile mf_open(MFile_area ma, const char *name, int block_size, int wflag)
365 char tmp[FILENAME_MAX+1];
368 yaz_log(YLOG_DEBUG, "mf_open(%s bs=%d, %s)", name, block_size,
369 wflag ? "RW" : "RDONLY");
371 for (mnew = ma->mfiles; mnew; mnew = mnew->next)
372 if (!strcmp(name, mnew->name))
376 yaz_log(YLOG_WARN, "metafile %s already open", name);
382 mnew = (meta_file *) xmalloc(sizeof(*mnew));
383 strcpy(mnew->name, name);
384 /* allocate one, empty file */
385 zebra_mutex_init(&mnew->mutex);
387 mnew->files[0].bytes = 0;
388 mnew->files[0].blocks = 0;
389 mnew->files[0].top = -1;
390 mnew->files[0].number = 0;
391 mnew->files[0].fd = -1;
392 mnew->min_bytes_creat = MF_MIN_BLOCKS_CREAT * block_size;
393 for (dp = ma->dirs; dp && dp->max_bytes >= 0 && dp->avail_bytes <
394 mnew->min_bytes_creat; dp = dp->next);
397 yaz_log(YLOG_FATAL, "Insufficient space for new mfile.");
400 mnew->files[0].dir = dp;
401 sprintf(tmp, "%s/%s-%d.mf", dp->name, mnew->name, 0);
402 mnew->files[0].path = xstrdup(tmp);
404 mnew->next = ma->mfiles;
409 for (i = 0; i < mnew->no_files; i++)
411 if (mnew->files[i].bytes % block_size)
412 mnew->files[i].bytes += block_size - mnew->files[i].bytes %
414 mnew->files[i].blocks = (int) (mnew->files[i].bytes / block_size);
418 mnew->blocksize = block_size;
419 mnew->min_bytes_creat = MF_MIN_BLOCKS_CREAT * block_size;
424 for (i = 0; i < mnew->no_files; i++)
426 mnew->files[i].blocks = (int)(mnew->files[i].bytes / mnew->blocksize);
427 if (i == mnew->no_files - 1)
428 mnew->files[i].top = -1;
431 i ? (mnew->files[i-1].top + mnew->files[i].blocks)
432 : (mnew->files[i].blocks - 1);
437 /** \brief closes metafile
438 \param mf metafile handle
441 int mf_close(MFile mf)
445 yaz_log(YLOG_DEBUG, "mf_close(%s)", mf->name);
447 for (i = 0; i < mf->no_files; i++)
449 if (mf->files[i].fd >= 0)
452 fsync(mf->files[i].fd);
454 close(mf->files[i].fd);
455 mf->files[i].fd = -1;
462 /** \brief reads block from metafile
463 \param mf metafile handle
464 \param no block position
465 \param offset offset within block
466 \param nbytes no of bytes to read (0 for whole block)
467 \param buf content (filled with data if OK)
468 \retval 0 block partially read
469 \retval 1 block fully read
470 \retval -1 block could not be read due to error
472 int mf_read(MFile mf, zint no, int offset, int nbytes, void *buf)
477 zebra_mutex_lock(&mf->mutex);
478 if ((rd = file_position(mf, no, offset)) < 0)
482 zebra_mutex_unlock(&mf->mutex);
487 yaz_log(YLOG_FATAL, "mf_read2 %s internal error", mf->name);
491 toread = nbytes ? nbytes : mf->blocksize;
492 if ((rd = read(mf->files[mf->cur_file].fd, buf, toread)) < 0)
494 yaz_log(YLOG_FATAL|YLOG_ERRNO, "mf_read2: Read failed (%s)",
495 mf->files[mf->cur_file].path);
498 zebra_mutex_unlock(&mf->mutex);
506 /** \brief writes block to metafile
507 \param mf metafile handle
508 \param no block position
509 \param offset offset within block
510 \param nbytes no of bytes to write (0 for whole block)
511 \param buf content to be written
512 \retval 0 block written
513 \retval -1 error (block not written)
515 int mf_write(MFile mf, zint no, int offset, int nbytes, const void *buf)
521 char tmp[FILENAME_MAX+1];
522 unsigned char dummych = '\xff';
524 zebra_mutex_lock(&mf->mutex);
525 if ((ps = file_position(mf, no, offset)) < 0)
527 yaz_log(YLOG_FATAL, "mf_write %s internal error (1)", mf->name);
530 /* file needs to grow */
531 while (ps >= mf->files[mf->cur_file].blocks)
533 mfile_off_t needed = (ps - mf->files[mf->cur_file].blocks + 1) *
535 /* file overflow - allocate new file */
536 if (mf->files[mf->cur_file].dir->max_bytes >= 0 &&
537 needed > mf->files[mf->cur_file].dir->avail_bytes)
540 if ((nblocks = (int) (mf->files[mf->cur_file].dir->avail_bytes /
543 yaz_log(YLOG_DEBUG, "Capping off file %s at pos " ZINT_FORMAT,
544 mf->files[mf->cur_file].path, nblocks);
545 if ((ps = file_position(mf,
546 (mf->cur_file ? mf->files[mf->cur_file-1].top : 0) +
547 mf->files[mf->cur_file].blocks + nblocks - 1, 0)) < 0)
549 yaz_log(YLOG_FATAL, "mf_write %s internal error (2)",
553 yaz_log(YLOG_DEBUG, "ps = " ZINT_FORMAT, ps);
554 if (write(mf->files[mf->cur_file].fd, &dummych, 1) < 1)
556 yaz_log(YLOG_ERRNO|YLOG_FATAL, "mf_write %s internal error (3)",
560 mf->files[mf->cur_file].blocks += nblocks;
561 mf->files[mf->cur_file].bytes += nblocks * mf->blocksize;
562 mf->files[mf->cur_file].dir->avail_bytes -= nblocks *
566 yaz_log(YLOG_DEBUG, "Creating new file.");
567 for (dp = mf->ma->dirs; dp && dp->max_bytes >= 0 &&
568 dp->avail_bytes < needed; dp = dp->next);
571 yaz_log(YLOG_FATAL, "Cannot allocate more space for %s",
575 mf->files[mf->cur_file].top = (mf->cur_file ?
576 mf->files[mf->cur_file-1].top : -1) +
577 mf->files[mf->cur_file].blocks;
578 mf->files[++(mf->cur_file)].top = -1;
579 mf->files[mf->cur_file].dir = dp;
580 mf->files[mf->cur_file].number =
581 mf->files[mf->cur_file-1].number + 1;
582 mf->files[mf->cur_file].blocks = 0;
583 mf->files[mf->cur_file].bytes = 0;
584 mf->files[mf->cur_file].fd = -1;
585 sprintf(tmp, "%s/%s-" ZINT_FORMAT ".mf", dp->name, mf->name,
586 mf->files[mf->cur_file].number);
587 mf->files[mf->cur_file].path = xstrdup(tmp);
589 /* open new file and position at beginning */
590 if ((ps = file_position(mf, no, offset)) < 0)
592 yaz_log(YLOG_FATAL, "mf_write %s internal error (4)",
599 nblocks = ps - mf->files[mf->cur_file].blocks + 1;
600 mf->files[mf->cur_file].blocks += nblocks;
601 mf->files[mf->cur_file].bytes += nblocks * mf->blocksize;
602 if (mf->files[mf->cur_file].dir->max_bytes >= 0)
603 mf->files[mf->cur_file].dir->avail_bytes -=
604 nblocks * mf->blocksize;
607 towrite = nbytes ? nbytes : mf->blocksize;
608 if (write(mf->files[mf->cur_file].fd, buf, towrite) < towrite)
610 yaz_log(YLOG_FATAL|YLOG_ERRNO, "Write failed for file %s part %d",
611 mf->name, mf->cur_file);
614 zebra_mutex_unlock(&mf->mutex);
618 /** \brief metafile area statistics
619 \param ma metafile area handle
620 \param no area number (0=first, 1=second, ..)
621 \param directory holds directory upon completion (if non-NULL)
622 \param used_bytes holds used bytes upon completion (if non-NULL)
623 \param max_bytes holds max size bytes upon completion (if non-NULL)
624 \retval 0 area number does not exist
625 \retval 1 area number exists (and directory,used_bytes,.. are set)
627 int mf_area_directory_stat(MFile_area ma, int no, const char **directory,
628 double *used_bytes, double *max_bytes)
631 mf_dir *d = ma->dirs;
632 for (i = 0; d && i<no; i++, d = d->next)
637 *directory = d->name;
640 /* possible loss of data. But it's just statistics and lies */
641 *max_bytes = (double) d->max_bytes;
645 /* possible loss of data. But it's just statistics and lies */
646 *used_bytes = (double) (d->max_bytes - d->avail_bytes);
653 * indent-tabs-mode: nil
655 * vim: shiftwidth=4 tabstop=8 expandtab