Pass init implementation + authentication stuff through filter virt_db.
[metaproxy-moved-to-github.git] / src / filter_virt_db.cpp
1 /* $Id: filter_virt_db.cpp,v 1.32 2006-01-18 10:30:28 adam Exp $
2    Copyright (c) 2005, Index Data.
3
4 %LICENSE%
5  */
6
7 #include "config.hpp"
8
9 #include "filter.hpp"
10 #include "package.hpp"
11
12 #include <boost/thread/mutex.hpp>
13 #include <boost/thread/condition.hpp>
14 #include <boost/shared_ptr.hpp>
15
16 #include "util.hpp"
17 #include "filter_virt_db.hpp"
18
19 #include <yaz/zgdu.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/diagbib1.h>
22
23 #include <map>
24 #include <iostream>
25
26 namespace yf = yp2::filter;
27
28 namespace yp2 {
29     namespace filter {
30
31         struct Virt_db::Set {
32             Set(BackendPtr b, std::string setname);
33             Set();
34             ~Set();
35
36             BackendPtr m_backend;
37             std::string m_setname;
38         };
39         struct Virt_db::Map {
40             Map(std::list<std::string> targets, std::string route);
41             Map();
42             std::list<std::string> m_targets;
43             std::string m_route;
44         };
45         struct Virt_db::Backend {
46             yp2::Session m_backend_session;
47             std::list<std::string> m_frontend_databases;
48             std::list<std::string> m_targets;
49             std::string m_route;
50             bool m_named_result_sets;
51             int m_number_of_sets;
52         };
53         struct Virt_db::Frontend {
54             Frontend(Rep *rep);
55             ~Frontend();
56             yp2::Session m_session;
57             bool m_is_virtual;
58             bool m_in_use;
59             yazpp_1::GDU m_init_gdu;
60             std::list<BackendPtr> m_backend_list;
61             std::map<std::string,Virt_db::Set> m_sets;
62
63             void search(Package &package, Z_APDU *apdu);
64             void present(Package &package, Z_APDU *apdu);
65             void scan(Package &package, Z_APDU *apdu);
66
67             void close(Package &package);
68             typedef std::map<std::string,Virt_db::Set>::iterator Sets_it;
69
70             BackendPtr lookup_backend_from_databases(
71                 std::list<std::string> databases);
72             BackendPtr create_backend_from_databases(
73                 std::list<std::string> databases,
74                 int &error_code,
75                 std::string &failing_database);
76             
77             BackendPtr init_backend(std::list<std::string> database,
78                                     Package &package,
79                                     int &error_code, std::string &addinfo);
80             Rep *m_p;
81         };            
82         class Virt_db::Rep {
83             friend class Virt_db;
84             friend class Frontend;
85             
86             FrontendPtr get_frontend(Package &package);
87             void release_frontend(Package &package);
88         private:
89             boost::mutex m_sessions_mutex;
90             std::map<std::string, Virt_db::Map>m_maps;
91
92             typedef std::map<std::string,Virt_db::Set>::iterator Sets_it;
93
94             boost::mutex m_mutex;
95             boost::condition m_cond_session_ready;
96             std::map<yp2::Session, FrontendPtr> m_clients;
97         };
98     }
99 }
100
101 using namespace yp2;
102
103 yf::Virt_db::BackendPtr yf::Virt_db::Frontend::lookup_backend_from_databases(
104     std::list<std::string> databases)
105 {
106     std::list<BackendPtr>::const_iterator map_it;
107     map_it = m_backend_list.begin();
108     for (; map_it != m_backend_list.end(); map_it++)
109         if ((*map_it)->m_frontend_databases == databases)
110             return *map_it;
111     BackendPtr null;
112     return null;
113 }
114
115 yf::Virt_db::BackendPtr yf::Virt_db::Frontend::create_backend_from_databases(
116     std::list<std::string> databases, int &error_code, std::string &addinfo)
117 {
118     BackendPtr b(new Backend);
119     std::list<std::string>::const_iterator db_it = databases.begin();
120
121     b->m_number_of_sets = 0;
122     b->m_frontend_databases = databases;
123     b->m_named_result_sets = false;
124
125     bool first_route = true;
126
127     std::map<std::string,bool> targets_dedup;
128     for (; db_it != databases.end(); db_it++)
129     {
130         std::map<std::string, Virt_db::Map>::iterator map_it;
131         map_it = m_p->m_maps.find(*db_it);
132         if (map_it == m_p->m_maps.end())  // database not found
133         {
134             error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
135             addinfo = *db_it;
136             BackendPtr ptr;
137             return ptr;
138         }
139         std::list<std::string>::const_iterator t_it =
140             map_it->second.m_targets.begin();
141         for (; t_it != map_it->second.m_targets.end(); t_it++)
142             targets_dedup[*t_it] = true;
143
144         // see if we have a route conflict.
145         if (!first_route && b->m_route != map_it->second.m_route)
146         {
147             // we have a conflict.. 
148             error_code =  YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
149             BackendPtr ptr;
150             return ptr;
151         }
152         b->m_route = map_it->second.m_route;
153         first_route = false;
154     }
155     std::map<std::string,bool>::const_iterator tm_it = targets_dedup.begin();
156     for (; tm_it != targets_dedup.end(); tm_it++)
157         b->m_targets.push_back(tm_it->first);
158
159     return b;
160 }
161
162 yf::Virt_db::BackendPtr yf::Virt_db::Frontend::init_backend(
163     std::list<std::string> databases, Package &package,
164     int &error_code, std::string &addinfo)
165 {
166     BackendPtr b = create_backend_from_databases(databases, error_code,
167                                                  addinfo);
168     if (!b)
169         return b;
170     Package init_package(b->m_backend_session, package.origin());
171     init_package.copy_filter(package);
172
173     yp2::odr odr;
174
175     Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
176
177     std::list<std::string>::const_iterator t_it = b->m_targets.begin();
178     int cat = 1;
179     for (; t_it != b->m_targets.end(); t_it++, cat++)
180     {
181         yaz_oi_set_string_oidval(&init_apdu->u.initRequest->otherInfo, odr,
182                                  VAL_PROXY, cat, t_it->c_str());
183     }
184         
185     Z_InitRequest *req = init_apdu->u.initRequest;
186
187     // copy stuff from Frontend Init Request
188     Z_GDU *org_gdu = m_init_gdu.get();
189     Z_InitRequest *org_init = org_gdu->u.z3950->u.initRequest;
190
191     req->idAuthentication = org_init->idAuthentication;
192     req->implementationId = org_init->implementationId;
193     req->implementationName = org_init->implementationName;
194     req->implementationVersion = org_init->implementationVersion;
195
196     ODR_MASK_SET(req->options, Z_Options_search);
197     ODR_MASK_SET(req->options, Z_Options_present);
198     ODR_MASK_SET(req->options, Z_Options_namedResultSets);
199     ODR_MASK_SET(req->options, Z_Options_scan);
200
201     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
202     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
203     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
204
205     init_package.request() = init_apdu;
206     
207     init_package.move(b->m_route);  // sending init 
208
209     Z_GDU *gdu = init_package.response().get();
210     // we hope to get an init response
211     if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
212         Z_APDU_initResponse)
213     {
214         Z_InitResponse *res = gdu->u.z3950->u.initResponse;
215         if (ODR_MASK_GET(res->options, Z_Options_namedResultSets))
216         {
217             b->m_named_result_sets = true;
218         }
219         std::cout << "GOT INIT res=" << *res->result << "\n";
220         if (!*res->result)
221         {
222             yp2::util::get_init_diagnostics(res, error_code, addinfo);
223             BackendPtr null;
224             return null; 
225         }
226     }
227     else
228     {
229         error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
230         // addinfo = database;
231         BackendPtr null;
232         return null;
233     }        
234     if (init_package.session().is_closed())
235     {
236         error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
237         // addinfo = database;
238         BackendPtr null;
239         return null;
240     }
241
242     m_backend_list.push_back(b);
243     return b;
244 }
245
246 void yf::Virt_db::Frontend::search(Package &package, Z_APDU *apdu_req)
247 {
248     Z_SearchRequest *req = apdu_req->u.searchRequest;
249     std::string vhost;
250     std::string resultSetId = req->resultSetName;
251     yp2::odr odr;
252
253     std::list<std::string> databases;
254     int i;
255     for (i = 0; i<req->num_databaseNames; i++)
256         databases.push_back(req->databaseNames[i]);
257
258     BackendPtr b; // null for now
259     Sets_it sets_it = m_sets.find(req->resultSetName);
260     if (sets_it != m_sets.end())
261     {
262         // result set already exist 
263         // if replace indicator is off: we return diagnostic if
264         // result set already exist.
265         if (*req->replaceIndicator == 0)
266         {
267             Z_APDU *apdu = 
268                 odr.create_searchResponse(
269                     apdu_req,
270                     YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
271                     0);
272             package.response() = apdu;
273             
274             return;
275         } 
276         sets_it->second.m_backend->m_number_of_sets--;
277
278         // pick up any existing backend with a database match
279         std::list<BackendPtr>::const_iterator map_it;
280         map_it = m_backend_list.begin();
281         for (; map_it != m_backend_list.end(); map_it++)
282         {
283             BackendPtr tmp = *map_it;
284             if (tmp->m_frontend_databases == databases)
285                 break;
286         }
287         if (map_it != m_backend_list.end()) 
288             b = *map_it;
289     }
290     else
291     {
292         // new result set.
293
294         // pick up any existing database with named result sets ..
295         // or one which has no result sets.. yet.
296         std::list<BackendPtr>::const_iterator map_it;
297         map_it = m_backend_list.begin();
298         for (; map_it != m_backend_list.end(); map_it++)
299         {
300             BackendPtr tmp = *map_it;
301             if (tmp->m_frontend_databases == databases &&
302                 (tmp->m_named_result_sets ||
303                  tmp->m_number_of_sets == 0))
304                 break;
305         }
306         if (map_it != m_backend_list.end()) 
307             b = *map_it;
308     }
309     if (!b)  // no backend yet. Must create a new one
310     {
311         int error_code;
312         std::string addinfo;
313         b = init_backend(databases, package, error_code, addinfo);
314         if (!b)
315         {
316             // did not get a backend (unavailable somehow?)
317             
318             Z_APDU *apdu = 
319                 odr.create_searchResponse(
320                     apdu_req, error_code, addinfo.c_str());
321             package.response() = apdu;
322             return;
323         }
324     }
325     m_sets.erase(req->resultSetName);
326     // sending search to backend
327     Package search_package(b->m_backend_session, package.origin());
328
329     search_package.copy_filter(package);
330
331     std::string backend_setname;
332     if (b->m_named_result_sets)
333     {
334         backend_setname = std::string(req->resultSetName);
335     }
336     else
337     {
338         backend_setname = "default";
339         req->resultSetName = odr_strdup(odr, backend_setname.c_str());
340     }
341
342     // pick first targets spec and move the databases from it ..
343     std::list<std::string>::const_iterator t_it = b->m_targets.begin();
344     if (t_it != b->m_targets.end())
345     {
346         if (!yp2::util::set_databases_from_zurl(odr, *t_it,
347                                                 &req->num_databaseNames,
348                                                 &req->databaseNames));
349     }
350
351     *req->replaceIndicator = 1;
352
353     search_package.request() = yazpp_1::GDU(apdu_req);
354     
355     search_package.move(b->m_route);
356
357     if (search_package.session().is_closed())
358     {
359         package.response() = search_package.response();
360         package.session().close();
361         return;
362     }
363     package.response() = search_package.response();
364
365     b->m_number_of_sets++;
366
367     m_sets[resultSetId] = Virt_db::Set(b, backend_setname);
368 }
369
370 yf::Virt_db::Frontend::Frontend(Rep *rep)
371 {
372     m_p = rep;
373     m_is_virtual = false;
374 }
375
376 void yf::Virt_db::Frontend::close(Package &package)
377 {
378     std::list<BackendPtr>::const_iterator b_it;
379     
380     for (b_it = m_backend_list.begin(); b_it != m_backend_list.end(); b_it++)
381     {
382         (*b_it)->m_backend_session.close();
383         Package close_package((*b_it)->m_backend_session, package.origin());
384         close_package.copy_filter(package);
385         close_package.move((*b_it)->m_route);
386     }
387     m_backend_list.clear();
388 }
389
390 yf::Virt_db::Frontend::~Frontend()
391 {
392 }
393
394 yf::Virt_db::FrontendPtr yf::Virt_db::Rep::get_frontend(Package &package)
395 {
396     boost::mutex::scoped_lock lock(m_mutex);
397
398     std::map<yp2::Session,yf::Virt_db::FrontendPtr>::iterator it;
399     
400     while(true)
401     {
402         it = m_clients.find(package.session());
403         if (it == m_clients.end())
404             break;
405         
406         if (!it->second->m_in_use)
407         {
408             it->second->m_in_use = true;
409             return it->second;
410         }
411         m_cond_session_ready.wait(lock);
412     }
413     FrontendPtr f(new Frontend(this));
414     m_clients[package.session()] = f;
415     f->m_in_use = true;
416     return f;
417 }
418
419 void yf::Virt_db::Rep::release_frontend(Package &package)
420 {
421     boost::mutex::scoped_lock lock(m_mutex);
422     std::map<yp2::Session,yf::Virt_db::FrontendPtr>::iterator it;
423     
424     it = m_clients.find(package.session());
425     if (it != m_clients.end())
426     {
427         if (package.session().is_closed())
428         {
429             it->second->close(package);
430             m_clients.erase(it);
431         }
432         else
433         {
434             it->second->m_in_use = false;
435         }
436         m_cond_session_ready.notify_all();
437     }
438 }
439
440 yf::Virt_db::Set::Set(BackendPtr b, std::string setname)
441     :  m_backend(b), m_setname(setname)
442 {
443 }
444
445
446 yf::Virt_db::Set::Set()
447 {
448 }
449
450
451 yf::Virt_db::Set::~Set()
452 {
453 }
454
455 yf::Virt_db::Map::Map(std::list<std::string> targets, std::string route)
456     : m_targets(targets), m_route(route) 
457 {
458 }
459
460 yf::Virt_db::Map::Map()
461 {
462 }
463
464 yf::Virt_db::Virt_db() : m_p(new Virt_db::Rep)
465 {
466 }
467
468 yf::Virt_db::~Virt_db() {
469 }
470
471 void yf::Virt_db::Frontend::present(Package &package, Z_APDU *apdu_req)
472 {
473     Z_PresentRequest *req = apdu_req->u.presentRequest;
474     std::string resultSetId = req->resultSetId;
475     yp2::odr odr;
476
477     Sets_it sets_it = m_sets.find(resultSetId);
478     if (sets_it == m_sets.end())
479     {
480         Z_APDU *apdu = 
481             odr.create_presentResponse(
482                 apdu_req,
483                 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
484                 resultSetId.c_str());
485         package.response() = apdu;
486         return;
487     }
488     Session *id =
489         new yp2::Session(sets_it->second.m_backend->m_backend_session);
490     
491     // sending present to backend
492     Package present_package(*id, package.origin());
493     present_package.copy_filter(package);
494
495     req->resultSetId = odr_strdup(odr, sets_it->second.m_setname.c_str());
496     
497     present_package.request() = yazpp_1::GDU(apdu_req);
498
499     present_package.move(sets_it->second.m_backend->m_route);
500
501     if (present_package.session().is_closed())
502     {
503         package.response() = present_package.response();
504         package.session().close();
505         return;
506     }
507     else
508     {
509         package.response() = present_package.response();
510     }
511     delete id;
512 }
513
514 void yf::Virt_db::Frontend::scan(Package &package, Z_APDU *apdu_req)
515 {
516     Z_ScanRequest *req = apdu_req->u.scanRequest;
517     std::string vhost;
518     yp2::odr odr;
519
520     std::list<std::string> databases;
521     int i;
522     for (i = 0; i<req->num_databaseNames; i++)
523         databases.push_back(req->databaseNames[i]);
524
525     BackendPtr b;
526     // pick up any existing backend with a database match
527     std::list<BackendPtr>::const_iterator map_it;
528     map_it = m_backend_list.begin();
529     for (; map_it != m_backend_list.end(); map_it++)
530     {
531         BackendPtr tmp = *map_it;
532         if (tmp->m_frontend_databases == databases)
533             break;
534     }
535     if (map_it != m_backend_list.end()) 
536         b = *map_it;
537     if (!b)  // no backend yet. Must create a new one
538     {
539         int error_code;
540         std::string addinfo;
541         b = init_backend(databases, package, error_code, addinfo);
542         if (!b)
543         {
544             // did not get a backend (unavailable somehow?)
545             Z_APDU *apdu =
546                 odr.create_scanResponse(
547                     apdu_req, error_code, addinfo.c_str());
548             package.response() = apdu;
549             
550             return;
551         }
552     }
553     // sending scan to backend
554     Package scan_package(b->m_backend_session, package.origin());
555
556     scan_package.copy_filter(package);
557
558     // pick first targets spec and move the databases from it ..
559     std::list<std::string>::const_iterator t_it = b->m_targets.begin();
560     if (t_it != b->m_targets.end())
561     {
562         if (!yp2::util::set_databases_from_zurl(odr, *t_it,
563                                                 &req->num_databaseNames,
564                                                 &req->databaseNames));
565     }
566     scan_package.request() = yazpp_1::GDU(apdu_req);
567     
568     scan_package.move(b->m_route);
569
570     if (scan_package.session().is_closed())
571     {
572         package.response() = scan_package.response();
573         package.session().close();
574         return;
575     }
576     package.response() = scan_package.response();
577 }
578
579
580 void yf::Virt_db::add_map_db2targets(std::string db, 
581                                      std::list<std::string> targets,
582                                      std::string route)
583 {
584     m_p->m_maps[db] = Virt_db::Map(targets, route);
585 }
586
587
588 void yf::Virt_db::add_map_db2target(std::string db, 
589                                     std::string target,
590                                     std::string route)
591 {
592     std::list<std::string> targets;
593     targets.push_back(target);
594
595     m_p->m_maps[db] = Virt_db::Map(targets, route);
596 }
597
598 void yf::Virt_db::process(Package &package) const
599 {
600     FrontendPtr f = m_p->get_frontend(package);
601
602     Z_GDU *gdu = package.request().get();
603     
604     if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
605         Z_APDU_initRequest && !f->m_is_virtual)
606     {
607         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
608         
609         const char *vhost =
610             yaz_oi_get_string_oidval(&req->otherInfo, VAL_PROXY, 1, 0);
611         if (!vhost)
612         {
613             f->m_init_gdu = gdu;
614             
615             yp2::odr odr;
616             Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
617             Z_InitResponse *resp = apdu->u.initResponse;
618             
619             int i;
620             static const int masks[] = {
621                 Z_Options_search,
622                 Z_Options_present,
623                 Z_Options_namedResultSets,
624                 Z_Options_scan,
625                 -1 
626             };
627             for (i = 0; masks[i] != -1; i++)
628                 if (ODR_MASK_GET(req->options, masks[i]))
629                     ODR_MASK_SET(resp->options, masks[i]);
630             
631             static const int versions[] = {
632                 Z_ProtocolVersion_1,
633                 Z_ProtocolVersion_2,
634                 Z_ProtocolVersion_3,
635                 -1
636             };
637             for (i = 0; versions[i] != -1; i++)
638                 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
639                     ODR_MASK_SET(resp->protocolVersion, versions[i]);
640                 else
641                     break;
642             
643             package.response() = apdu;
644             f->m_is_virtual = true;
645         }
646         else
647             package.move();
648     }
649     else if (!f->m_is_virtual)
650         package.move();
651     else if (gdu && gdu->which == Z_GDU_Z3950)
652     {
653         Z_APDU *apdu = gdu->u.z3950;
654         if (apdu->which == Z_APDU_initRequest)
655         {
656             yp2::odr odr;
657             
658             package.response() = odr.create_close(
659                 apdu,
660                 Z_Close_protocolError,
661                 "double init");
662             
663             package.session().close();
664         }
665         else if (apdu->which == Z_APDU_searchRequest)
666         {
667             f->search(package, apdu);
668         }
669         else if (apdu->which == Z_APDU_presentRequest)
670         {
671             f->present(package, apdu);
672         }
673         else if (apdu->which == Z_APDU_scanRequest)
674         {
675             f->scan(package, apdu);
676         }
677         else
678         {
679             yp2::odr odr;
680             
681             package.response() = odr.create_close(
682                 apdu, Z_Close_protocolError,
683                 "unsupported APDU in filter_virt_db");
684             
685             package.session().close();
686         }
687     }
688     m_p->release_frontend(package);
689 }
690
691
692 void yp2::filter::Virt_db::configure(const xmlNode * ptr)
693 {
694     for (ptr = ptr->children; ptr; ptr = ptr->next)
695     {
696         if (ptr->type != XML_ELEMENT_NODE)
697             continue;
698         if (!strcmp((const char *) ptr->name, "virtual"))
699         {
700             std::string database;
701             std::list<std::string> targets;
702             xmlNode *v_node = ptr->children;
703             for (; v_node; v_node = v_node->next)
704             {
705                 if (v_node->type != XML_ELEMENT_NODE)
706                     continue;
707                 
708                 if (yp2::xml::is_element_yp2(v_node, "database"))
709                     database = yp2::xml::get_text(v_node);
710                 else if (yp2::xml::is_element_yp2(v_node, "target"))
711                     targets.push_back(yp2::xml::get_text(v_node));
712                 else
713                     throw yp2::filter::FilterException
714                         ("Bad element " 
715                          + std::string((const char *) v_node->name)
716                          + " in virtual section"
717                             );
718             }
719             std::string route = yp2::xml::get_route(ptr);
720             add_map_db2targets(database, targets, route);
721         }
722         else
723         {
724             throw yp2::filter::FilterException
725                 ("Bad element " 
726                  + std::string((const char *) ptr->name)
727                  + " in virt_db filter");
728         }
729     }
730 }
731
732 static yp2::filter::Base* filter_creator()
733 {
734     return new yp2::filter::Virt_db;
735 }
736
737 extern "C" {
738     struct yp2_filter_struct yp2_filter_virt_db = {
739         0,
740         "virt_db",
741         filter_creator
742     };
743 }
744
745
746 /*
747  * Local variables:
748  * c-basic-offset: 4
749  * indent-tabs-mode: nil
750  * c-file-style: "stroustrup"
751  * End:
752  * vim: shiftwidth=4 tabstop=8 expandtab
753  */