Avoid debug std::cout
[metaproxy-moved-to-github.git] / src / filter_virt_db.cpp
1 /* $Id: filter_virt_db.cpp,v 1.34 2006-02-02 10:25:13 adam Exp $
2    Copyright (c) 2005, Index Data.
3
4 %LICENSE%
5  */
6
7 #include "config.hpp"
8
9 #include "filter.hpp"
10 #include "package.hpp"
11
12 #include <boost/thread/mutex.hpp>
13 #include <boost/thread/condition.hpp>
14 #include <boost/shared_ptr.hpp>
15
16 #include "util.hpp"
17 #include "filter_virt_db.hpp"
18
19 #include <yaz/zgdu.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/diagbib1.h>
22
23 #include <map>
24 #include <iostream>
25
26 namespace yf = yp2::filter;
27
28 namespace yp2 {
29     namespace filter {
30
31         struct Virt_db::Set {
32             Set(BackendPtr b, std::string setname);
33             Set();
34             ~Set();
35
36             BackendPtr m_backend;
37             std::string m_setname;
38         };
39         struct Virt_db::Map {
40             Map(std::list<std::string> targets, std::string route);
41             Map();
42             std::list<std::string> m_targets;
43             std::string m_route;
44         };
45         struct Virt_db::Backend {
46             yp2::Session m_backend_session;
47             std::list<std::string> m_frontend_databases;
48             std::list<std::string> m_targets;
49             std::string m_route;
50             bool m_named_result_sets;
51             int m_number_of_sets;
52         };
53         struct Virt_db::Frontend {
54             Frontend(Rep *rep);
55             ~Frontend();
56             yp2::Session m_session;
57             bool m_is_virtual;
58             bool m_in_use;
59             yazpp_1::GDU m_init_gdu;
60             std::list<BackendPtr> m_backend_list;
61             std::map<std::string,Virt_db::Set> m_sets;
62
63             void search(Package &package, Z_APDU *apdu);
64             void present(Package &package, Z_APDU *apdu);
65             void scan(Package &package, Z_APDU *apdu);
66
67             void close(Package &package);
68             typedef std::map<std::string,Virt_db::Set>::iterator Sets_it;
69
70             BackendPtr lookup_backend_from_databases(
71                 std::list<std::string> databases);
72             BackendPtr create_backend_from_databases(
73                 std::list<std::string> databases,
74                 int &error_code,
75                 std::string &failing_database);
76             
77             BackendPtr init_backend(std::list<std::string> database,
78                                     Package &package,
79                                     int &error_code, std::string &addinfo);
80             Rep *m_p;
81         };            
82         class Virt_db::Rep {
83             friend class Virt_db;
84             friend class Frontend;
85             
86             FrontendPtr get_frontend(Package &package);
87             void release_frontend(Package &package);
88         private:
89             boost::mutex m_sessions_mutex;
90             std::map<std::string, Virt_db::Map>m_maps;
91
92             typedef std::map<std::string,Virt_db::Set>::iterator Sets_it;
93
94             boost::mutex m_mutex;
95             boost::condition m_cond_session_ready;
96             std::map<yp2::Session, FrontendPtr> m_clients;
97         };
98     }
99 }
100
101 using namespace yp2;
102
103 yf::Virt_db::BackendPtr yf::Virt_db::Frontend::lookup_backend_from_databases(
104     std::list<std::string> databases)
105 {
106     std::list<BackendPtr>::const_iterator map_it;
107     map_it = m_backend_list.begin();
108     for (; map_it != m_backend_list.end(); map_it++)
109         if ((*map_it)->m_frontend_databases == databases)
110             return *map_it;
111     BackendPtr null;
112     return null;
113 }
114
115 yf::Virt_db::BackendPtr yf::Virt_db::Frontend::create_backend_from_databases(
116     std::list<std::string> databases, int &error_code, std::string &addinfo)
117 {
118     BackendPtr b(new Backend);
119     std::list<std::string>::const_iterator db_it = databases.begin();
120
121     b->m_number_of_sets = 0;
122     b->m_frontend_databases = databases;
123     b->m_named_result_sets = false;
124
125     bool first_route = true;
126
127     std::map<std::string,bool> targets_dedup;
128     for (; db_it != databases.end(); db_it++)
129     {
130         std::map<std::string, Virt_db::Map>::iterator map_it;
131         map_it = m_p->m_maps.find(*db_it);
132         if (map_it == m_p->m_maps.end())  // database not found
133         {
134             error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
135             addinfo = *db_it;
136             BackendPtr ptr;
137             return ptr;
138         }
139         std::list<std::string>::const_iterator t_it =
140             map_it->second.m_targets.begin();
141         for (; t_it != map_it->second.m_targets.end(); t_it++)
142             targets_dedup[*t_it] = true;
143
144         // see if we have a route conflict.
145         if (!first_route && b->m_route != map_it->second.m_route)
146         {
147             // we have a conflict.. 
148             error_code =  YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
149             BackendPtr ptr;
150             return ptr;
151         }
152         b->m_route = map_it->second.m_route;
153         first_route = false;
154     }
155     std::map<std::string,bool>::const_iterator tm_it = targets_dedup.begin();
156     for (; tm_it != targets_dedup.end(); tm_it++)
157         b->m_targets.push_back(tm_it->first);
158
159     return b;
160 }
161
162 yf::Virt_db::BackendPtr yf::Virt_db::Frontend::init_backend(
163     std::list<std::string> databases, Package &package,
164     int &error_code, std::string &addinfo)
165 {
166     BackendPtr b = create_backend_from_databases(databases, error_code,
167                                                  addinfo);
168     if (!b)
169         return b;
170     Package init_package(b->m_backend_session, package.origin());
171     init_package.copy_filter(package);
172
173     yp2::odr odr;
174
175     Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
176
177     yp2::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, odr,
178                                    b->m_targets);
179     Z_InitRequest *req = init_apdu->u.initRequest;
180
181     // copy stuff from Frontend Init Request
182     Z_GDU *org_gdu = m_init_gdu.get();
183     Z_InitRequest *org_init = org_gdu->u.z3950->u.initRequest;
184
185     req->idAuthentication = org_init->idAuthentication;
186     req->implementationId = org_init->implementationId;
187     req->implementationName = org_init->implementationName;
188     req->implementationVersion = org_init->implementationVersion;
189
190     ODR_MASK_SET(req->options, Z_Options_search);
191     ODR_MASK_SET(req->options, Z_Options_present);
192     ODR_MASK_SET(req->options, Z_Options_namedResultSets);
193     ODR_MASK_SET(req->options, Z_Options_scan);
194
195     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
196     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
197     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
198
199     init_package.request() = init_apdu;
200     
201     init_package.move(b->m_route);  // sending init 
202
203     Z_GDU *gdu = init_package.response().get();
204     // we hope to get an init response
205     if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
206         Z_APDU_initResponse)
207     {
208         Z_InitResponse *res = gdu->u.z3950->u.initResponse;
209         if (ODR_MASK_GET(res->options, Z_Options_namedResultSets))
210         {
211             b->m_named_result_sets = true;
212         }
213         if (!*res->result)
214         {
215             yp2::util::get_init_diagnostics(res, error_code, addinfo);
216             BackendPtr null;
217             return null; 
218         }
219     }
220     else
221     {
222         error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
223         // addinfo = database;
224         BackendPtr null;
225         return null;
226     }        
227     if (init_package.session().is_closed())
228     {
229         error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
230         // addinfo = database;
231         BackendPtr null;
232         return null;
233     }
234
235     m_backend_list.push_back(b);
236     return b;
237 }
238
239 void yf::Virt_db::Frontend::search(Package &package, Z_APDU *apdu_req)
240 {
241     Z_SearchRequest *req = apdu_req->u.searchRequest;
242     std::string vhost;
243     std::string resultSetId = req->resultSetName;
244     yp2::odr odr;
245
246     std::list<std::string> databases;
247     int i;
248     for (i = 0; i<req->num_databaseNames; i++)
249         databases.push_back(req->databaseNames[i]);
250
251     BackendPtr b; // null for now
252     Sets_it sets_it = m_sets.find(req->resultSetName);
253     if (sets_it != m_sets.end())
254     {
255         // result set already exist 
256         // if replace indicator is off: we return diagnostic if
257         // result set already exist.
258         if (*req->replaceIndicator == 0)
259         {
260             Z_APDU *apdu = 
261                 odr.create_searchResponse(
262                     apdu_req,
263                     YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
264                     0);
265             package.response() = apdu;
266             
267             return;
268         } 
269         sets_it->second.m_backend->m_number_of_sets--;
270
271         // pick up any existing backend with a database match
272         std::list<BackendPtr>::const_iterator map_it;
273         map_it = m_backend_list.begin();
274         for (; map_it != m_backend_list.end(); map_it++)
275         {
276             BackendPtr tmp = *map_it;
277             if (tmp->m_frontend_databases == databases)
278                 break;
279         }
280         if (map_it != m_backend_list.end()) 
281             b = *map_it;
282     }
283     else
284     {
285         // new result set.
286
287         // pick up any existing database with named result sets ..
288         // or one which has no result sets.. yet.
289         std::list<BackendPtr>::const_iterator map_it;
290         map_it = m_backend_list.begin();
291         for (; map_it != m_backend_list.end(); map_it++)
292         {
293             BackendPtr tmp = *map_it;
294             if (tmp->m_frontend_databases == databases &&
295                 (tmp->m_named_result_sets ||
296                  tmp->m_number_of_sets == 0))
297                 break;
298         }
299         if (map_it != m_backend_list.end()) 
300             b = *map_it;
301     }
302     if (!b)  // no backend yet. Must create a new one
303     {
304         int error_code;
305         std::string addinfo;
306         b = init_backend(databases, package, error_code, addinfo);
307         if (!b)
308         {
309             // did not get a backend (unavailable somehow?)
310             
311             Z_APDU *apdu = 
312                 odr.create_searchResponse(
313                     apdu_req, error_code, addinfo.c_str());
314             package.response() = apdu;
315             return;
316         }
317     }
318     m_sets.erase(req->resultSetName);
319     // sending search to backend
320     Package search_package(b->m_backend_session, package.origin());
321
322     search_package.copy_filter(package);
323
324     std::string backend_setname;
325     if (b->m_named_result_sets)
326     {
327         backend_setname = std::string(req->resultSetName);
328     }
329     else
330     {
331         backend_setname = "default";
332         req->resultSetName = odr_strdup(odr, backend_setname.c_str());
333     }
334
335     // pick first targets spec and move the databases from it ..
336     std::list<std::string>::const_iterator t_it = b->m_targets.begin();
337     if (t_it != b->m_targets.end())
338     {
339         if (!yp2::util::set_databases_from_zurl(odr, *t_it,
340                                                 &req->num_databaseNames,
341                                                 &req->databaseNames));
342     }
343
344     *req->replaceIndicator = 1;
345
346     search_package.request() = yazpp_1::GDU(apdu_req);
347     
348     search_package.move(b->m_route);
349
350     if (search_package.session().is_closed())
351     {
352         package.response() = search_package.response();
353         package.session().close();
354         return;
355     }
356     package.response() = search_package.response();
357
358     b->m_number_of_sets++;
359
360     m_sets[resultSetId] = Virt_db::Set(b, backend_setname);
361 }
362
363 yf::Virt_db::Frontend::Frontend(Rep *rep)
364 {
365     m_p = rep;
366     m_is_virtual = false;
367 }
368
369 void yf::Virt_db::Frontend::close(Package &package)
370 {
371     std::list<BackendPtr>::const_iterator b_it;
372     
373     for (b_it = m_backend_list.begin(); b_it != m_backend_list.end(); b_it++)
374     {
375         (*b_it)->m_backend_session.close();
376         Package close_package((*b_it)->m_backend_session, package.origin());
377         close_package.copy_filter(package);
378         close_package.move((*b_it)->m_route);
379     }
380     m_backend_list.clear();
381 }
382
383 yf::Virt_db::Frontend::~Frontend()
384 {
385 }
386
387 yf::Virt_db::FrontendPtr yf::Virt_db::Rep::get_frontend(Package &package)
388 {
389     boost::mutex::scoped_lock lock(m_mutex);
390
391     std::map<yp2::Session,yf::Virt_db::FrontendPtr>::iterator it;
392     
393     while(true)
394     {
395         it = m_clients.find(package.session());
396         if (it == m_clients.end())
397             break;
398         
399         if (!it->second->m_in_use)
400         {
401             it->second->m_in_use = true;
402             return it->second;
403         }
404         m_cond_session_ready.wait(lock);
405     }
406     FrontendPtr f(new Frontend(this));
407     m_clients[package.session()] = f;
408     f->m_in_use = true;
409     return f;
410 }
411
412 void yf::Virt_db::Rep::release_frontend(Package &package)
413 {
414     boost::mutex::scoped_lock lock(m_mutex);
415     std::map<yp2::Session,yf::Virt_db::FrontendPtr>::iterator it;
416     
417     it = m_clients.find(package.session());
418     if (it != m_clients.end())
419     {
420         if (package.session().is_closed())
421         {
422             it->second->close(package);
423             m_clients.erase(it);
424         }
425         else
426         {
427             it->second->m_in_use = false;
428         }
429         m_cond_session_ready.notify_all();
430     }
431 }
432
433 yf::Virt_db::Set::Set(BackendPtr b, std::string setname)
434     :  m_backend(b), m_setname(setname)
435 {
436 }
437
438
439 yf::Virt_db::Set::Set()
440 {
441 }
442
443
444 yf::Virt_db::Set::~Set()
445 {
446 }
447
448 yf::Virt_db::Map::Map(std::list<std::string> targets, std::string route)
449     : m_targets(targets), m_route(route) 
450 {
451 }
452
453 yf::Virt_db::Map::Map()
454 {
455 }
456
457 yf::Virt_db::Virt_db() : m_p(new Virt_db::Rep)
458 {
459 }
460
461 yf::Virt_db::~Virt_db() {
462 }
463
464 void yf::Virt_db::Frontend::present(Package &package, Z_APDU *apdu_req)
465 {
466     Z_PresentRequest *req = apdu_req->u.presentRequest;
467     std::string resultSetId = req->resultSetId;
468     yp2::odr odr;
469
470     Sets_it sets_it = m_sets.find(resultSetId);
471     if (sets_it == m_sets.end())
472     {
473         Z_APDU *apdu = 
474             odr.create_presentResponse(
475                 apdu_req,
476                 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
477                 resultSetId.c_str());
478         package.response() = apdu;
479         return;
480     }
481     Session *id =
482         new yp2::Session(sets_it->second.m_backend->m_backend_session);
483     
484     // sending present to backend
485     Package present_package(*id, package.origin());
486     present_package.copy_filter(package);
487
488     req->resultSetId = odr_strdup(odr, sets_it->second.m_setname.c_str());
489     
490     present_package.request() = yazpp_1::GDU(apdu_req);
491
492     present_package.move(sets_it->second.m_backend->m_route);
493
494     if (present_package.session().is_closed())
495     {
496         package.response() = present_package.response();
497         package.session().close();
498         return;
499     }
500     else
501     {
502         package.response() = present_package.response();
503     }
504     delete id;
505 }
506
507 void yf::Virt_db::Frontend::scan(Package &package, Z_APDU *apdu_req)
508 {
509     Z_ScanRequest *req = apdu_req->u.scanRequest;
510     std::string vhost;
511     yp2::odr odr;
512
513     std::list<std::string> databases;
514     int i;
515     for (i = 0; i<req->num_databaseNames; i++)
516         databases.push_back(req->databaseNames[i]);
517
518     BackendPtr b;
519     // pick up any existing backend with a database match
520     std::list<BackendPtr>::const_iterator map_it;
521     map_it = m_backend_list.begin();
522     for (; map_it != m_backend_list.end(); map_it++)
523     {
524         BackendPtr tmp = *map_it;
525         if (tmp->m_frontend_databases == databases)
526             break;
527     }
528     if (map_it != m_backend_list.end()) 
529         b = *map_it;
530     if (!b)  // no backend yet. Must create a new one
531     {
532         int error_code;
533         std::string addinfo;
534         b = init_backend(databases, package, error_code, addinfo);
535         if (!b)
536         {
537             // did not get a backend (unavailable somehow?)
538             Z_APDU *apdu =
539                 odr.create_scanResponse(
540                     apdu_req, error_code, addinfo.c_str());
541             package.response() = apdu;
542             
543             return;
544         }
545     }
546     // sending scan to backend
547     Package scan_package(b->m_backend_session, package.origin());
548
549     scan_package.copy_filter(package);
550
551     // pick first targets spec and move the databases from it ..
552     std::list<std::string>::const_iterator t_it = b->m_targets.begin();
553     if (t_it != b->m_targets.end())
554     {
555         if (!yp2::util::set_databases_from_zurl(odr, *t_it,
556                                                 &req->num_databaseNames,
557                                                 &req->databaseNames));
558     }
559     scan_package.request() = yazpp_1::GDU(apdu_req);
560     
561     scan_package.move(b->m_route);
562
563     if (scan_package.session().is_closed())
564     {
565         package.response() = scan_package.response();
566         package.session().close();
567         return;
568     }
569     package.response() = scan_package.response();
570 }
571
572
573 void yf::Virt_db::add_map_db2targets(std::string db, 
574                                      std::list<std::string> targets,
575                                      std::string route)
576 {
577     m_p->m_maps[db] = Virt_db::Map(targets, route);
578 }
579
580
581 void yf::Virt_db::add_map_db2target(std::string db, 
582                                     std::string target,
583                                     std::string route)
584 {
585     std::list<std::string> targets;
586     targets.push_back(target);
587
588     m_p->m_maps[db] = Virt_db::Map(targets, route);
589 }
590
591 void yf::Virt_db::process(Package &package) const
592 {
593     FrontendPtr f = m_p->get_frontend(package);
594
595     Z_GDU *gdu = package.request().get();
596     
597     if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
598         Z_APDU_initRequest && !f->m_is_virtual)
599     {
600         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
601         
602         std::list<std::string> vhosts;
603         yp2::util::get_vhost_otherinfo(&req->otherInfo, false, vhosts);
604         if (vhosts.size() == 0)
605         {
606             f->m_init_gdu = gdu;
607             
608             yp2::odr odr;
609             Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
610             Z_InitResponse *resp = apdu->u.initResponse;
611             
612             int i;
613             static const int masks[] = {
614                 Z_Options_search,
615                 Z_Options_present,
616                 Z_Options_namedResultSets,
617                 Z_Options_scan,
618                 -1 
619             };
620             for (i = 0; masks[i] != -1; i++)
621                 if (ODR_MASK_GET(req->options, masks[i]))
622                     ODR_MASK_SET(resp->options, masks[i]);
623             
624             static const int versions[] = {
625                 Z_ProtocolVersion_1,
626                 Z_ProtocolVersion_2,
627                 Z_ProtocolVersion_3,
628                 -1
629             };
630             for (i = 0; versions[i] != -1; i++)
631                 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
632                     ODR_MASK_SET(resp->protocolVersion, versions[i]);
633                 else
634                     break;
635             
636             package.response() = apdu;
637             f->m_is_virtual = true;
638         }
639         else
640             package.move();
641     }
642     else if (!f->m_is_virtual)
643         package.move();
644     else if (gdu && gdu->which == Z_GDU_Z3950)
645     {
646         Z_APDU *apdu = gdu->u.z3950;
647         if (apdu->which == Z_APDU_initRequest)
648         {
649             yp2::odr odr;
650             
651             package.response() = odr.create_close(
652                 apdu,
653                 Z_Close_protocolError,
654                 "double init");
655             
656             package.session().close();
657         }
658         else if (apdu->which == Z_APDU_searchRequest)
659         {
660             f->search(package, apdu);
661         }
662         else if (apdu->which == Z_APDU_presentRequest)
663         {
664             f->present(package, apdu);
665         }
666         else if (apdu->which == Z_APDU_scanRequest)
667         {
668             f->scan(package, apdu);
669         }
670         else
671         {
672             yp2::odr odr;
673             
674             package.response() = odr.create_close(
675                 apdu, Z_Close_protocolError,
676                 "unsupported APDU in filter_virt_db");
677             
678             package.session().close();
679         }
680     }
681     m_p->release_frontend(package);
682 }
683
684
685 void yp2::filter::Virt_db::configure(const xmlNode * ptr)
686 {
687     for (ptr = ptr->children; ptr; ptr = ptr->next)
688     {
689         if (ptr->type != XML_ELEMENT_NODE)
690             continue;
691         if (!strcmp((const char *) ptr->name, "virtual"))
692         {
693             std::string database;
694             std::list<std::string> targets;
695             xmlNode *v_node = ptr->children;
696             for (; v_node; v_node = v_node->next)
697             {
698                 if (v_node->type != XML_ELEMENT_NODE)
699                     continue;
700                 
701                 if (yp2::xml::is_element_yp2(v_node, "database"))
702                     database = yp2::xml::get_text(v_node);
703                 else if (yp2::xml::is_element_yp2(v_node, "target"))
704                     targets.push_back(yp2::xml::get_text(v_node));
705                 else
706                     throw yp2::filter::FilterException
707                         ("Bad element " 
708                          + std::string((const char *) v_node->name)
709                          + " in virtual section"
710                             );
711             }
712             std::string route = yp2::xml::get_route(ptr);
713             add_map_db2targets(database, targets, route);
714         }
715         else
716         {
717             throw yp2::filter::FilterException
718                 ("Bad element " 
719                  + std::string((const char *) ptr->name)
720                  + " in virt_db filter");
721         }
722     }
723 }
724
725 static yp2::filter::Base* filter_creator()
726 {
727     return new yp2::filter::Virt_db;
728 }
729
730 extern "C" {
731     struct yp2_filter_struct yp2_filter_virt_db = {
732         0,
733         "virt_db",
734         filter_creator
735     };
736 }
737
738
739 /*
740  * Local variables:
741  * c-basic-offset: 4
742  * indent-tabs-mode: nil
743  * c-file-style: "stroustrup"
744  * End:
745  * vim: shiftwidth=4 tabstop=8 expandtab
746  */