-/* $Id: benchindex1.c,v 1.2 2006-12-10 21:02:28 adam Exp $
+/* $Id: benchindex1.c,v 1.4 2006-12-11 15:08:55 adam Exp $
Copyright (C) 1995-2006
Index Data ApS
*/
#include <yaz/options.h>
-#if HAVE_SYS_TIMES_H
-#include <sys/times.h>
-#endif
-#if HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
struct index_block {
NMEM nmem;
- size_t no_entries;
+ int no_entries;
size_t current_entry;
size_t current_max;
struct index_term *terms;
int no_words = 0, no_new_words = 0;
const char *dict_info = 0;
ISAM_P isamc_p = 0;
-
-#if HAVE_SYS_TIMES_H
-#if HAVE_SYS_TIME_H
- struct tms tms1, tms2;
- struct timeval start_time, end_time;
- double usec;
- times(&tms1);
- gettimeofday(&start_time, 0);
-#endif
-#endif
+ zebra_timing_t tim = zebra_timing_create();
b->ar = xmalloc(sizeof(*b->ar) * b->no_entries);
for (i = 0; i < b->no_entries; i++, t = t->next)
nmem_reset(b->nmem);
b->no_entries = 0;
b->terms = 0;
-
-#if HAVE_SYS_TIMES_H
-#if HAVE_SYS_TIME_H
b->round++;
- gettimeofday(&end_time, 0);
- times(&tms2);
-
- usec = (end_time.tv_sec - start_time.tv_sec) * 1000000.0 +
- end_time.tv_usec - start_time.tv_usec;
-
+
+ zebra_timing_stop(tim);
+
printf("%3d %8.6f %5.2f %5.2f\n",
b->round,
- usec / 1000000,
- (double) (tms2.tms_utime - tms1.tms_utime)/100,
- (double) (tms2.tms_stime - tms1.tms_stime)/100);
-#endif
-#endif
-
+ zebra_timing_get_real(tim),
+ zebra_timing_get_user(tim),
+ zebra_timing_get_sys(tim));
+ fflush(stdout);
+ zebra_timing_destroy(&tim);
}
void index_block_check_flush(struct index_block *b, ISAMB isb, Dict dict,
void exit_usage(void)
{
- fprintf(stderr, "benchindex1 [-m m] [iso2709file]\n");
+ fprintf(stderr, "benchindex1 [-t type] [-m mem] [-i] [inputfile]\n");
exit(1);
}
(*seqno)++;
}
-void index_wrbuf(struct index_block *b, WRBUF wrbuf, zint docid)
+void index_wrbuf(struct index_block *b, WRBUF wrbuf, zint docid,
+ int subfield_char)
{
int nl = 1;
const char *cp = wrbuf_buf(wrbuf);
if (nl)
{
int i;
- for (i = 0; i<6 && *cp; i++, cp++)
- ;
+ if (cp[0] != ' ')
+ { /* skip field+indicator (e.g. 245 00) */
+ for (i = 0; i<6 && *cp; i++, cp++)
+ ;
+ }
+ else
+ { /* continuation line */
+ for (i = 0; i<4 && *cp; i++, cp++)
+ ;
+ }
}
nl = 0;
if (*cp == '\n')
nl = 1;
cp++;
}
- else if (*cp == '$' && cp[1])
+ else if (*cp == subfield_char && cp[1])
{
if (sz)
{
}
cp += 2;
}
- else if (strchr("$/-;,.:[]\"&(){} ", *cp))
+ else if (strchr("$*/-;,.:[]\"&(){} ", *cp))
{
if (sz)
{
index_term(b, term, docid, &seqno);
}
+void index_marc_line_records(ISAMB isb,
+ Dict dict,
+ zint *docid_seq,
+ FILE *inf,
+ int memory)
+{
+ WRBUF wrbuf = wrbuf_alloc();
+ int no_docs = 0;
+ int new_rec = 1;
+ char line[4096];
+ struct index_block *b = index_block_new(memory);
+ while(fgets(line, sizeof(line)-1, inf))
+ {
+ if (line[0] == '$')
+ {
+ if (!new_rec)
+ new_rec = 1;
+ else
+ new_rec = 0;
+ continue;
+ }
+ if (new_rec)
+ {
+ (*docid_seq)++;
+ no_docs++;
+ index_block_check_flush(b, isb, dict, no_docs);
+ new_rec = 0;
+ }
+
+ if (line[0] == ' ')
+ {
+ /* continuation */
+ wrbuf_puts(wrbuf, line);
+ continue;
+ }
+ else
+ {
+ /* index existing buffer (if any) */
+ if (wrbuf_len(wrbuf))
+ {
+ index_wrbuf(b, wrbuf, *docid_seq, '*');
+ wrbuf_rewind(wrbuf);
+ }
+ if (line[0] != ' ' && line[1] != ' ' && line[2] != ' ' &&
+ line[3] == ' ')
+ {
+ /* normal field+indicator line */
+ wrbuf_puts(wrbuf, line);
+ }
+ }
+ }
+ if (wrbuf_len(wrbuf))
+ {
+ index_wrbuf(b, wrbuf, *docid_seq, '*');
+ wrbuf_rewind(wrbuf);
+ }
+ (*docid_seq)++;
+ no_docs++;
+ index_block_check_flush(b, isb, dict, no_docs);
+ index_block_destroy(&b);
+}
+
void index_marc_from_file(ISAMB isb,
Dict dict,
+ zint *docid_seq,
FILE *inf,
int memory,
int verbose, int print_offset)
yaz_marc_t mt = yaz_marc_create();
WRBUF wrbuf = wrbuf_alloc();
struct index_block *b = index_block_new(memory);
- const char *dict_info = 0;
- zint docid_seq = 1;
int no_docs = 0;
- dict_info = dict_lookup(dict, "_s");
- if (dict_info)
- {
- assert(*dict_info == sizeof(docid_seq));
- memcpy(&docid_seq, dict_info+1, sizeof(docid_seq));
- }
-
while (1)
{
size_t r;
if (yaz_marc_write_line(mt, wrbuf))
break;
- index_wrbuf(b, wrbuf, docid_seq);
+ index_wrbuf(b, wrbuf, *docid_seq, '$');
wrbuf_rewind(wrbuf);
- docid_seq++;
+ (*docid_seq)++;
no_docs++;
index_block_check_flush(b, isb, dict, no_docs);
wrbuf_free(wrbuf, 1);
yaz_marc_destroy(mt);
index_block_destroy(&b);
- yaz_log(YLOG_LOG, "Total " ZINT_FORMAT " documents", docid_seq);
- dict_insert(dict, "_s", sizeof(docid_seq), &docid_seq);
}
int main(int argc, char **argv)
int memory = 5;
const char *fname = 0;
FILE *inf = stdin;
+ zebra_timing_t tim = 0;
+ zint docid_seq = 1;
+ const char *dict_info;
+ const char *type = "iso2709";
- while ((ret = options("im:", argv, argc, &arg)) != -2)
+ while ((ret = options("im:t:", argv, argc, &arg)) != -2)
{
switch(ret)
{
case 'i':
reset = 1;
break;
+ case 't':
+ if (!strcmp(arg, "iso2709"))
+ type = "iso2709";
+ else if (!strcmp(arg, "line"))
+ type = "line";
+ else
+ {
+ fprintf(stderr, "bad type: %s.\n", arg);
+ exit_usage();
+ }
+ break;
case 0:
fname = arg;
break;
if (reset)
bf_reset(bfs);
+ tim = zebra_timing_create();
/* create isam handle */
isb = isamb_open (bfs, "isamb", 1, &method, 0);
if (!isb)
}
dict = dict_open(bfs, "dict", 50, 1, 0, 4096);
- index_marc_from_file(isb, dict, inf, memory,
- 0 /* verbose */ , 0 /* print_offset */);
+ dict_info = dict_lookup(dict, "_s");
+ if (dict_info)
+ {
+ assert(*dict_info == sizeof(docid_seq));
+ memcpy(&docid_seq, dict_info+1, sizeof(docid_seq));
+ }
+
+ if (!strcmp(type, "iso2709"))
+ index_marc_from_file(isb, dict, &docid_seq, inf, memory,
+ 0 /* verbose */ , 0 /* print_offset */);
+ else if (!strcmp(type, "line"))
+ index_marc_line_records(isb, dict, &docid_seq, inf, memory);
+
+ yaz_log(YLOG_LOG, "Total " ZINT_FORMAT " documents", docid_seq);
+ dict_insert(dict, "_s", sizeof(docid_seq), &docid_seq);
dict_close(dict);
isamb_close(isb);
fclose(inf);
/* exit block system */
bfs_destroy(bfs);
+ zebra_timing_stop(tim);
+
+ yaz_log(YLOG_LOG, "Total %8.6f %5.2f %5.2f\n",
+ zebra_timing_get_real(tim),
+ zebra_timing_get_user(tim),
+ zebra_timing_get_sys(tim));
+
+ zebra_timing_destroy(&tim);
+
exit(0);
return 0;
}
projects / idzebra-moved-to-github.git / blobdiff