From 48d26dcea283f57c49f77b82d6a987e41d775749 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 8 Jun 1998 14:43:09 +0000 Subject: [PATCH] Added suport for EXPLAIN Proxy servers - added settings databasePath and explainDatabase to facilitate this. Increased maximum number of databases and attributes in one register. --- CHANGELOG | 9 +++++++++ index/extract.c | 37 +++++++++++++++++++++++-------------- index/index.h | 10 +++++++++- index/kcompare.c | 22 +++++++++++++++++++++- index/main.c | 9 ++++++++- index/trav.c | 38 +++++++++++++++++++++++++++++--------- index/zinfo.c | 26 ++++++++++++++++++-------- index/zinfo.h | 10 ++++++++-- index/zrpn.c | 24 +++++++++++++++++++----- test/usmarc/zebra.cfg | 5 ++++- 10 files changed, 148 insertions(+), 42 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 73b7c05..98670c5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +Maximum number of databases in one Zebra register increased. + +New setting, databasePath, which specifies that first directory +during update traversal is the database name (instead of +a fixed one). + +New setting, explainDatabase, which specifies that databases +are EXPLAIN aware. + Modified Zebra so that it works with ASN.1 compiled code for YAZ. Implemented EXPLAIN database maintenance. Zebra automatically diff --git a/index/extract.c b/index/extract.c index f3bceae..b12769c 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.82 1998-05-20 10:12:15 adam + * Revision 1.83 1998-06-08 14:43:10 adam + * Added suport for EXPLAIN Proxy servers - added settings databasePath + * and explainDatabase to facilitate this. Increased maximum number + * of databases and attributes in one register. + * + * Revision 1.82 1998/05/20 10:12:15 adam * Implemented automatic EXPLAIN database maintenance. * Modified Zebra to work with ASN.1 compiled version of YAZ. * @@ -839,7 +844,8 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys) if (ch < 0) ch = zebraExplain_addSU (zti, attrSet, attrUse); assert (ch > 0); - ((char*) key_buf) [key_buf_used++] = ch; + key_buf_used += key_SU_code (ch, ((char*)key_buf) + key_buf_used); + while (*src) ((char*)key_buf) [key_buf_used++] = *src++; src++; @@ -1460,12 +1466,10 @@ int fileExtract (SYSNO *sysno, const char *fname, logf (LOG_DEBUG, "fileExtract %s", fname); /* determine file extension */ + *ext = '\0'; for (i = strlen(fname); --i >= 0; ) if (fname[i] == '/') - { - strcpy (ext, ""); break; - } else if (fname[i] == '.') { strcpy (ext, fname+i+1); @@ -1478,20 +1482,17 @@ int fileExtract (SYSNO *sysno, const char *fname, if (!(rGroup->recordType = res_get (common_resource, ext_res))) { sprintf (ext_res, "%srecordType", gprefix); - if (!(rGroup->recordType = res_get (common_resource, ext_res))) - { - if (records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "? %s", fname); - return 0; - } + rGroup->recordType = res_get (common_resource, ext_res); } } if (!rGroup->recordType) { if (records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "? record %s", fname); + logf (LOG_LOG, "? %s", fname); return 0; } + if (!*rGroup->recordType) + return 0; if (!(recType = recType_byName (rGroup->recordType, subType))) { logf (LOG_WARN, "No such record type: %s", rGroup->recordType); @@ -1518,9 +1519,17 @@ int fileExtract (SYSNO *sysno, const char *fname, if (!rGroup->databaseName) rGroup->databaseName = "Default"; + /* determine if explain database */ + + sprintf (ext_res, "%sexplainDatabase", gprefix); + rGroup->explainDatabase = + atoi (res_get_def (common_resource, ext_res, "0")); + + /* announce database */ if (zebraExplain_curDatabase (zti, rGroup->databaseName)) { - if (zebraExplain_newDatabase (zti, rGroup->databaseName)) + if (zebraExplain_newDatabase (zti, rGroup->databaseName, + rGroup->explainDatabase)) abort (); } @@ -1586,7 +1595,7 @@ static int explain_extract (void *handle, Record rec, data1_node *n) if (zebraExplain_curDatabase (zti, rec->info[recInfo_databaseName])) { - if (zebraExplain_newDatabase (zti, rec->info[recInfo_databaseName])) + if (zebraExplain_newDatabase (zti, rec->info[recInfo_databaseName], 0)) abort (); } diff --git a/index/index.h b/index/index.h index 688f792..6ba88da 100644 --- a/index/index.h +++ b/index/index.h @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: index.h,v $ - * Revision 1.58 1998-05-20 10:12:16 adam + * Revision 1.59 1998-06-08 14:43:11 adam + * Added suport for EXPLAIN Proxy servers - added settings databasePath + * and explainDatabase to facilitate this. Increased maximum number + * of databases and attributes in one register. + * + * Revision 1.58 1998/05/20 10:12:16 adam * Implemented automatic EXPLAIN database maintenance. * Modified Zebra to work with ASN.1 compiled version of YAZ. * @@ -256,6 +261,8 @@ struct recordGroup { int flagStoreKeys; int flagRw; int fileVerboseLimit; + int databaseNamePath; + int explainDatabase; data1_handle dh; BFiles bfs; ZebraMaps zebra_maps; @@ -293,6 +300,7 @@ void inv_prstat (BFiles bfs); void key_input (BFiles bfs, int nkeys, int cache); ISAMC_M key_isamc_m (Res res); int merge_sort (char **buf, int from, int to); +int key_SU_code (int ch, char *out); #define FNAME_DICT "dict" #define FNAME_ISAM "isam" diff --git a/index/kcompare.c b/index/kcompare.c index e259f77..64fcbfe 100644 --- a/index/kcompare.c +++ b/index/kcompare.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: kcompare.c,v $ - * Revision 1.23 1998-03-05 08:45:12 adam + * Revision 1.24 1998-06-08 14:43:12 adam + * Added suport for EXPLAIN Proxy servers - added settings databasePath + * and explainDatabase to facilitate this. Increased maximum number + * of databases and attributes in one register. + * + * Revision 1.23 1998/03/05 08:45:12 adam * New result set model and modular ranking system. Moved towards * descent server API. System information stored as "SGML" records. * @@ -286,3 +291,18 @@ ISAMC_M key_isamc_m (Res res) return me; } +int key_SU_code (int ch, char *out) +{ + int i; + logf (LOG_LOG, "key_SU_code: %d", ch); + for (i = 0; ch; i++) + { + if (ch > 63) + out[i] = 128 + (ch & 63); + else + out[i] = 1 + ch; + logf (LOG_LOG, " %d", out[i]); + ch = ch >> 6; + } + return i; +} diff --git a/index/main.c b/index/main.c index bda4eab..149c9c8 100644 --- a/index/main.c +++ b/index/main.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: main.c,v $ - * Revision 1.57 1998-05-20 10:12:18 adam + * Revision 1.58 1998-06-08 14:43:13 adam + * Added suport for EXPLAIN Proxy servers - added settings databasePath + * and explainDatabase to facilitate this. Increased maximum number + * of databases and attributes in one register. + * + * Revision 1.57 1998/05/20 10:12:18 adam * Implemented automatic EXPLAIN database maintenance. * Modified Zebra to work with ASN.1 compiled version of YAZ. * @@ -250,6 +255,8 @@ int main (int argc, char **argv) rGroupDef.flagStoreData = -1; rGroupDef.flagStoreKeys = -1; rGroupDef.flagRw = 1; + rGroupDef.databaseNamePath = 0; + rGroupDef.explainDatabase = 0; rGroupDef.fileVerboseLimit = 100000; rGroupDef.zebra_maps = NULL; rGroupDef.dh = data1_create (); diff --git a/index/trav.c b/index/trav.c index 49af41e..943b4f1 100644 --- a/index/trav.c +++ b/index/trav.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: trav.c,v $ - * Revision 1.33 1998-01-12 15:04:08 adam + * Revision 1.34 1998-06-08 14:43:14 adam + * Added suport for EXPLAIN Proxy servers - added settings databasePath + * and explainDatabase to facilitate this. Increased maximum number + * of databases and attributes in one register. + * + * Revision 1.33 1998/01/12 15:04:08 adam * The test option (-s) only uses read-lock (and not write lock). * * Revision 1.32 1997/09/25 14:56:51 adam @@ -149,7 +154,8 @@ static int repComp (const char *a, const char *b, size_t len) } static void repositoryExtractR (int deleteFlag, char *rep, - struct recordGroup *rGroup) + struct recordGroup *rGroup, + int level) { struct dir_entry *e; int i; @@ -163,16 +169,23 @@ static void repositoryExtractR (int deleteFlag, char *rep, rep[rep_len] = '/'; else --rep_len; + for (i=0; e[i].name; i++) { + char *ecp; strcpy (rep +rep_len+1, e[i].name); + if ((ecp = strrchr (e[i].name, '/'))) + *ecp = '\0'; + if (level == 0 && rGroup->databaseNamePath) + rGroup->databaseName = e[i].name; + switch (e[i].kind) { case dirs_file: fileExtract (NULL, rep, rGroup, deleteFlag); break; case dirs_dir: - repositoryExtractR (deleteFlag, rep, rGroup); + repositoryExtractR (deleteFlag, rep, rGroup, level+1); break; } } @@ -211,8 +224,9 @@ static void fileDeleteR (struct dirs_info *di, struct dirs_entry *dst, } static void fileUpdateR (struct dirs_info *di, struct dirs_entry *dst, - const char *base, char *src, - struct recordGroup *rGroup) + const char *base, char *src, + struct recordGroup *rGroup, + int level) { struct dir_entry *e_src; int i_src = 0; @@ -278,6 +292,9 @@ static void fileUpdateR (struct dirs_info *di, struct dirs_entry *dst, else break; logf (LOG_DEBUG, "trav sd=%d", sd); + + if (level == 0 && rGroup->databaseNamePath) + rGroup->databaseName = e_src[i_src].name; if (sd == 0) { strcpy (src + src_len, e_src[i_src].name); @@ -298,7 +315,7 @@ static void fileUpdateR (struct dirs_info *di, struct dirs_entry *dst, dst = dirs_read (di); break; case dirs_dir: - fileUpdateR (di, dst, base, src, rGroup); + fileUpdateR (di, dst, base, src, rGroup, level+1); dst = dirs_last (di); logf (LOG_DEBUG, "last is %s", dst ? dst->path : "null"); break; @@ -320,7 +337,7 @@ static void fileUpdateR (struct dirs_info *di, struct dirs_entry *dst, dirs_add (di, src, sysno, e_src[i_src].mtime); break; case dirs_dir: - fileUpdateR (di, dst, base, src, rGroup); + fileUpdateR (di, dst, base, src, rGroup, level+1); if (dst) dst = dirs_last (di); break; @@ -360,6 +377,9 @@ static void groupRes (struct recordGroup *rGroup) sprintf (resStr, "%srecordId", gPrefix); rGroup->recordId = res_get (common_resource, resStr); + sprintf (resStr, "%sdatabasePath", gPrefix); + rGroup->databaseNamePath = + atoi (res_get_def (common_resource, resStr, "0")); } void repositoryShow (struct recordGroup *rGroup) @@ -437,7 +457,7 @@ static void fileUpdate (Dict dict, struct recordGroup *rGroup, } di = dirs_open (dict, src, rGroup->flagRw); *dst = '\0'; - fileUpdateR (di, dirs_read (di), src, dst, rGroup); + fileUpdateR (di, dirs_read (di), src, dst, rGroup, 0); dirs_free (&di); } else @@ -460,7 +480,7 @@ static void repositoryExtract (int deleteFlag, struct recordGroup *rGroup, if (S_ISREG(sbuf.st_mode)) fileExtract (NULL, src, rGroup, deleteFlag); else if (S_ISDIR(sbuf.st_mode)) - repositoryExtractR (deleteFlag, src, rGroup); + repositoryExtractR (deleteFlag, src, rGroup, 0); else logf (LOG_WARN, "Ignoring path %s", src); } diff --git a/index/zinfo.c b/index/zinfo.c index 41c6dd9..2e6dae9 100644 --- a/index/zinfo.c +++ b/index/zinfo.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zinfo.c,v $ - * Revision 1.9 1998-06-02 12:10:27 adam + * Revision 1.10 1998-06-08 14:43:15 adam + * Added suport for EXPLAIN Proxy servers - added settings databasePath + * and explainDatabase to facilitate this. Increased maximum number + * of databases and attributes in one register. + * + * Revision 1.9 1998/06/02 12:10:27 adam * Fixed bug related to attributeDetails. * * Revision 1.8 1998/05/20 10:12:20 adam @@ -556,7 +561,7 @@ ZebraExplainInfo zebraExplain_open ( } *zdip = NULL; rec_rm (&trec); - zebraExplain_newDatabase (zei, "IR-Explain-1"); + zebraExplain_newDatabase (zei, "IR-Explain-1", 0); } return zei; } @@ -734,7 +739,8 @@ static void zebraExplain_updateAccessInfo (ZebraExplainInfo zei, data1_node *n, } } -int zebraExplain_newDatabase (ZebraExplainInfo zei, const char *database) +int zebraExplain_newDatabase (ZebraExplainInfo zei, const char *database, + int explain_database) { struct zebDatabaseInfoB *zdi; data1_node *node_dbinfo, *node_adinfo; @@ -779,13 +785,17 @@ int zebraExplain_newDatabase (ZebraExplainInfo zei, const char *database) data1_add_tagdata_text (zei->dh, node_dbinfo, "name", database, zei->nmem); - + + if (explain_database) + data1_add_tagdata_text (zei->dh, node_dbinfo, "explainDatabase", + "", zei->nmem); + data1_add_tagdata_text (zei->dh, node_dbinfo, "userFee", - "0", zei->nmem); - + "0", zei->nmem); + data1_add_tagdata_text (zei->dh, node_dbinfo, "available", - "1", zei->nmem); - + "1", zei->nmem); + #if ZINFO_DEBUG data1_pr_tree (zei->dh, zdi->data1_database, stderr); #endif diff --git a/index/zinfo.h b/index/zinfo.h index f50d7a1..f0ede54 100644 --- a/index/zinfo.h +++ b/index/zinfo.h @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zinfo.h,v $ - * Revision 1.4 1998-05-20 10:12:21 adam + * Revision 1.5 1998-06-08 14:43:16 adam + * Added suport for EXPLAIN Proxy servers - added settings databasePath + * and explainDatabase to facilitate this. Increased maximum number + * of databases and attributes in one register. + * + * Revision 1.4 1998/05/20 10:12:21 adam * Implemented automatic EXPLAIN database maintenance. * Modified Zebra to work with ASN.1 compiled version of YAZ. * @@ -37,7 +42,8 @@ ZebraExplainInfo zebraExplain_open (Records records, data1_handle dh, void zebraExplain_close (ZebraExplainInfo zei, int writeFlag, int (*updateH)(Record drec, data1_node *n)); int zebraExplain_curDatabase (ZebraExplainInfo zei, const char *database); -int zebraExplain_newDatabase (ZebraExplainInfo zei, const char *database); +int zebraExplain_newDatabase (ZebraExplainInfo zei, const char *database, + int explain_database); int zebraExplain_lookupSU (ZebraExplainInfo zei, int set, int use); int zebraExplain_addSU (ZebraExplainInfo zei, int set, int use); void zebraExplain_addSchema (ZebraExplainInfo zei, Odr_oid *oid); diff --git a/index/zrpn.c b/index/zrpn.c index 4c6641e..07f0091 100644 --- a/index/zrpn.c +++ b/index/zrpn.c @@ -4,7 +4,12 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zrpn.c,v $ - * Revision 1.77 1998-05-20 10:12:22 adam + * Revision 1.78 1998-06-08 14:43:17 adam + * Added suport for EXPLAIN Proxy servers - added settings databasePath + * and explainDatabase to facilitate this. Increased maximum number + * of databases and attributes in one register. + * + * Revision 1.77 1998/05/20 10:12:22 adam * Implemented automatic EXPLAIN database maintenance. * Modified Zebra to work with ASN.1 compiled version of YAZ. * @@ -435,9 +440,10 @@ static void add_isam_p (const char *name, const char *info, assert (*info == sizeof(*p->isam_p_buf)); memcpy (p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf)); +#if 0 term_untrans (p->zh, p->reg_type, term_tmp, name+2); logf (LOG_DEBUG, "grep: %s", term_tmp); - +#endif (p->isam_p_indx)++; } @@ -829,6 +835,8 @@ static int field_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt, local_attr = local_attr->next) { int ord; + char ord_buf[32]; + int i, ord_len; ord = zebraExplain_lookupSU (zh->zei, attp.attset_ordinal, local_attr->local); @@ -838,8 +846,13 @@ static int field_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt, term_dict[prefix_len++] = '|'; else term_dict[prefix_len++] = '('; - term_dict[prefix_len++] = 1; - term_dict[prefix_len++] = ord; + + ord_len = key_SU_code (ord, ord_buf); + for (i = 0; ilist)); for (j = 0; jlist[j].term = NULL; - termz[prefix_len++] = ords[i]; + + prefix_len += key_SU_code (ords[i], termz + prefix_len); termz[prefix_len++] = reg_id; termz[prefix_len] = 0; strcpy (scan_info->prefix, termz); diff --git a/test/usmarc/zebra.cfg b/test/usmarc/zebra.cfg index 897085c..3e01731 100644 --- a/test/usmarc/zebra.cfg +++ b/test/usmarc/zebra.cfg @@ -1,6 +1,6 @@ # Simple Zebra configuration file that defines # a database with USMARC records. -# $Id: zebra.cfg,v 1.2 1998-05-20 10:12:30 adam Exp $ +# $Id: zebra.cfg,v 1.3 1998-06-08 14:43:18 adam Exp $ # # Where are the YAZ / Zebra tables located? profilePath: .:../../tab:../../../yaz/tab @@ -27,3 +27,6 @@ keyTmpDir: tmp # Approx. Memory usage during indexing memMax: 4 +# Shadow Area +shadow: shadow:100M + -- 1.7.10.4