zoom: tune CCL to Z39.50 diagnostics mapping
[metaproxy-moved-to-github.git] / src / filter_zoom.cpp
index 17d7c10..97a1fde 100644 (file)
@@ -93,8 +93,10 @@ namespace metaproxy_1 {
             xsltStylesheetPtr xsp;
             std::string content_session_id;
             bool enable_cproxy;
+            bool enable_explain;
+            xmlDoc *explain_doc;
         public:
-            Backend(SearchablePtr sptr);
+            Backend();
             ~Backend();
             void connect(std::string zurl, int *error, char **addinfo,
                          ODR odr);
@@ -116,12 +118,20 @@ namespace metaproxy_1 {
             BackendPtr m_backend;
             void handle_package(mp::Package &package);
             void handle_search(mp::Package &package);
+
+            BackendPtr explain_search(mp::Package &package,
+                                      std::string &database,
+                                      int *error,
+                                      char **addinfo,
+                                      mp::odr &odr,
+                                      std::string &torus_db,
+                                      std::string &realm);
             void handle_present(mp::Package &package);
             BackendPtr get_backend_from_databases(mp::Package &package,
                                                   std::string &database,
                                                   int *error,
                                                   char **addinfo,
-                                                  ODR odr,
+                                                  mp::odr &odr,
                                                   int *proxy_step);
 
             bool create_content_session(mp::Package &package,
@@ -149,6 +159,15 @@ namespace metaproxy_1 {
                                    ODR odr, BackendPtr b,
                                    Odr_oid *preferredRecordSyntax,
                                    const char *element_set_name);
+            Z_Records *get_explain_records(Package &package,
+                                           Odr_int start,
+                                           Odr_int number_to_present,
+                                           int *error,
+                                           char **addinfo,
+                                           Odr_int *number_of_records_returned,
+                                           ODR odr, BackendPtr b,
+                                           Odr_oid *preferredRecordSyntax,
+                                           const char *element_set_name);
 
             void log_diagnostic(mp::Package &package,
                                 int error, const char *addinfo);
@@ -186,11 +205,35 @@ namespace metaproxy_1 {
             std::string element_transform;
             std::string element_raw;
             std::string proxy;
+            xsltStylesheetPtr explain_xsp;
             std::map<std::string,SearchablePtr> s_map;
         };
     }
 }
 
+
+static xmlNode *xml_node_search(xmlNode *ptr, int *num, int m)
+{
+    while (ptr)
+    {
+        if (ptr->type == XML_ELEMENT_NODE &&
+            !strcmp((const char *) ptr->name, "recordData"))
+        {
+            (*num)++;
+            if (m == *num)
+                return ptr;
+        }
+        else  // else: we don't want to find nested nodes
+        {   
+            xmlNode *ret_node = xml_node_search(ptr->children, num, m);
+            if (ret_node)
+                return ret_node;
+        }
+        ptr = ptr->next;
+    }
+    return 0;
+}
+
 // define Pimpl wrapper forwarding to Impl
  
 yf::Zoom::Zoom() : m_p(new Impl)
@@ -215,7 +258,7 @@ void yf::Zoom::process(mp::Package &package) const
 
 // define Implementation stuff
 
-yf::Zoom::Backend::Backend(SearchablePtr ptr) : sptr(ptr)
+yf::Zoom::Backend::Backend()
 {
     m_apdu_wrbuf = wrbuf_alloc();
     m_connection = ZOOM_connection_create(0);
@@ -223,14 +266,19 @@ yf::Zoom::Backend::Backend(SearchablePtr ptr) : sptr(ptr)
     m_resultset = 0;
     xsp = 0;
     enable_cproxy = true;
+    enable_explain = false;
+    explain_doc = 0;
 }
 
 yf::Zoom::Backend::~Backend()
 {
     if (xsp)
         xsltFreeStylesheet(xsp);
+    if (explain_doc)
+        xmlFreeDoc(explain_doc);
     ZOOM_connection_destroy(m_connection);
     ZOOM_resultset_destroy(m_resultset);
+    wrbuf_destroy(m_apdu_wrbuf);
 }
 
 
@@ -278,6 +326,7 @@ void yf::Zoom::Backend::connect(std::string zurl,
 void yf::Zoom::Backend::search(ZOOM_query q, Odr_int *hits,
                                int *error, char **addinfo, ODR odr)
 {
+    ZOOM_resultset_destroy(m_resultset);
     m_resultset = ZOOM_connection_search(m_connection, q);
     get_zoom_error(error, addinfo, odr);
     if (*error == 0)
@@ -384,11 +433,14 @@ yf::Zoom::Impl::Impl() :
 {
     bibset = ccl_qual_mk();
 
+    explain_xsp = 0;
     srand((unsigned int) time(0));
 }
 
 yf::Zoom::Impl::~Impl()
-{ 
+{
+    if (explain_xsp)
+        xsltFreeStylesheet(explain_xsp);
     ccl_qual_rm(&bibset);
 }
 
@@ -493,8 +545,11 @@ yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
                           "cclmap_", 7))
         {
             std::string value = mp::xml::get_text(ptr);
-            ccl_qual_fitem(s->ccl_bibset, value.c_str(),
-                           (const char *) ptr->name + 7);
+            if (value.length() > 0)
+            {
+                ccl_qual_fitem(s->ccl_bibset, value.c_str(),
+                               (const char *) ptr->name + 7);
+            }
         }
         else if (!strncmp((const char *) ptr->name,
                           "sortmap_", 8))
@@ -562,6 +617,8 @@ void yf::Zoom::Impl::configure_local_records(const xmlNode *ptr, bool test_only)
 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
                                const char *path)
 {
+    std::string explain_xslt_fname;
+
     content_tmp_file = "/tmp/cf.XXXXXX.p";
     if (path && *path)
     {
@@ -590,6 +647,8 @@ void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
                     element_raw = mp::xml::get_text(attr->children);
                 else if (!strcmp((const char *) attr->name, "proxy"))
                     proxy = mp::xml::get_text(attr->children);
+                else if (!strcmp((const char *) attr->name, "explain_xsl"))
+                    explain_xslt_fname = mp::xml::get_text(attr->children);
                 else
                     throw mp::filter::FilterException(
                         "Bad attribute " + std::string((const char *)
@@ -661,6 +720,41 @@ void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
                  + " in zoom filter");
         }
     }
+
+    if (explain_xslt_fname.length())
+    {
+        const char *path = 0;
+        
+        if (xsldir.length())
+            path = xsldir.c_str();
+        else
+            path = file_path.c_str();
+        
+        char fullpath[1024];
+        char *cp = yaz_filepath_resolve(explain_xslt_fname.c_str(),
+                                        path, 0, fullpath);
+        if (!cp)
+        {
+            throw mp::filter::FilterException
+                ("Cannot read XSLT " + explain_xslt_fname);
+        }
+
+        xmlDoc *xsp_doc = xmlParseFile(cp);
+        if (!xsp_doc)
+        {
+            throw mp::filter::FilterException
+                ("Cannot parse XSLT " + explain_xslt_fname);
+        }
+
+        explain_xsp = xsltParseStylesheetDoc(xsp_doc);
+        if (!explain_xsp)
+        {
+            xmlFreeDoc(xsp_doc);
+            throw mp::filter::FilterException
+                ("Cannot parse XSLT " + explain_xslt_fname);
+            
+        }
+    }
 }
 
 bool yf::Zoom::Frontend::create_content_session(mp::Package &package,
@@ -710,17 +804,19 @@ bool yf::Zoom::Frontend::create_content_session(mp::Package &package,
         fclose(file);
         package.log("zoom", YLOG_LOG, "content file: %s", fname);
         xfree(fname);
+        wrbuf_destroy(w);
     }
     return true;
 }
 
 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     mp::Package &package,
-    std::string &database, int *error, char **addinfo, ODR odr,
+    std::string &database, int *error, char **addinfo, mp::odr &odr,
     int *proxy_step)
 {
     std::list<BackendPtr>::const_iterator map_it;
-    if (m_backend && m_backend->m_frontend_database == database)
+    if (m_backend && !m_backend->enable_explain && 
+        m_backend->m_frontend_database == database)
         return m_backend;
 
     std::string input_args;
@@ -784,7 +880,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
         {
             char **dstr;
             int dnum = 0;
-            nmem_strsplit(odr->mem, ",", value, &dstr, &dnum);
+            nmem_strsplit(((ODR) odr)->mem, ",", value, &dstr, &dnum);
             if (*proxy_step >= dnum)
                 *proxy_step = 0;
             else
@@ -831,6 +927,11 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
         if (param_content_password)
             content_authentication += "/" + std::string(param_content_password);
     }
+
+    if (torus_db.compare("IR-Explain---1") == 0)
+        return explain_search(package, database, error, addinfo, odr, torus_db,
+            realm);
+    
     SearchablePtr sptr;
 
     std::map<std::string,SearchablePtr>::iterator it;
@@ -839,7 +940,9 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
         sptr = it->second;
     else if (torus_url.length() > 0)
     {
+        std::string torus_query = "udb==" + torus_db;
         xmlDoc *doc = mp::get_searchable(package,torus_url, torus_db,
+                                         torus_query,
                                          realm, m_p->proxy);
         if (!doc)
         {
@@ -952,8 +1055,9 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
 
     m_backend.reset();
 
-    BackendPtr b(new Backend(sptr));
+    BackendPtr b(new Backend);
 
+    b->sptr = sptr;
     b->xsp = xsp;
     b->m_frontend_database = database;
     b->enable_cproxy = param_nocproxy ? false : true;
@@ -1130,6 +1234,66 @@ void yf::Zoom::Frontend::prepare_elements(BackendPtr b,
         b->set_option("schema", element_set_name);
 }
 
+Z_Records *yf::Zoom::Frontend::get_explain_records(
+    Package &package,
+    Odr_int start,
+    Odr_int number_to_present,
+    int *error,
+    char **addinfo,
+    Odr_int *number_of_records_returned,
+    ODR odr,
+    BackendPtr b,
+    Odr_oid *preferredRecordSyntax,
+    const char *element_set_name)
+{
+    Odr_int i;
+    Z_Records *records = 0;
+
+    if (!b->explain_doc)
+    {
+        return records;
+    }
+    if (number_to_present > 10000)
+        number_to_present = 10000;
+
+    xmlNode *ptr = xmlDocGetRootElement(b->explain_doc);
+    
+    Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
+        odr_malloc(odr, sizeof(*npl));
+    npl->records = (Z_NamePlusRecord **)
+        odr_malloc(odr, number_to_present * sizeof(*npl->records));
+    
+    for (i = 0; i < number_to_present; i++)
+    {
+        int num = 0;
+        xmlNode *res = xml_node_search(ptr, &num, start + i + 1);
+        if (!res)
+            break;
+        xmlBufferPtr xml_buf = xmlBufferCreate();
+        xmlNode *tmp_node = xmlCopyNode(res->children, 1);
+        xmlNodeDump(xml_buf, tmp_node->doc, tmp_node, 0, 0);
+
+        Z_NamePlusRecord *npr =
+            (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
+        npr->databaseName = odr_strdup(odr, b->m_frontend_database.c_str());
+        npr->which = Z_NamePlusRecord_databaseRecord;
+        npr->u.databaseRecord =
+            z_ext_record_xml(odr,
+                             (const char *) xml_buf->content, xml_buf->use);
+        npl->records[i] = npr;
+        xmlFreeNode(tmp_node);
+        xmlBufferFree(xml_buf);
+    }
+    records = (Z_Records*) odr_malloc(odr, sizeof(*records));
+    records->which = Z_Records_DBOSD;
+    records->u.databaseOrSurDiagnostics = npl;
+
+    npl->num_records = i;
+    *number_of_records_returned = i;
+    return records;
+}
+
+
 Z_Records *yf::Zoom::Frontend::get_records(Package &package,
                                            Odr_int start,
                                            Odr_int number_to_present,
@@ -1190,8 +1354,6 @@ Z_Records *yf::Zoom::Frontend::get_records(Package &package,
             Z_NamePlusRecord *npr = 0;
             const char *addinfo;
 
-            package.log("zoom", YLOG_LOG, "Inspecting record at position %d",
-                        start + i);
             int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
                                               &addinfo, 0 /* diagset */);
                 
@@ -1412,6 +1574,79 @@ void yf::Zoom::Frontend::log_diagnostic(mp::Package &package,
                     error, err_msg);
 }
 
+yf::Zoom::BackendPtr yf::Zoom::Frontend::explain_search(mp::Package &package,
+                                                        std::string &database,
+                                                        int *error,
+                                                        char **addinfo,
+                                                        mp::odr &odr,
+                                                        std::string &torus_db,
+                                                        std::string &realm)
+{
+    m_backend.reset();
+
+    BackendPtr b(new Backend);
+
+    b->m_frontend_database = database;
+    b->enable_explain = true;
+   
+    Z_GDU *gdu = package.request().get();
+    Z_APDU *apdu_req = gdu->u.z3950;
+    Z_SearchRequest *sr = apdu_req->u.searchRequest;
+    Z_Query *query = sr->query;
+
+    if (!m_p->explain_xsp)
+    {
+        *error = YAZ_BIB1_UNSPECIFIED_ERROR;
+        *addinfo =
+            odr_strdup(odr, "IR-Explain---1 unsupported. torus explain_xsl not defined");
+        return m_backend;
+    }
+    else if (query->which == Z_Query_type_104 &&
+        query->u.type_104->which == Z_External_CQL)
+    {
+        std::string torus_url = m_p->torus_searchable_url;
+        std::string torus_query(query->u.type_104->u.cql);
+        xmlDoc *doc = mp::get_searchable(package, torus_url, "",
+                                         torus_query,
+                                         realm, m_p->proxy);
+        if (m_p->explain_xsp)
+        {
+            xmlDoc *rec_res =  xsltApplyStylesheet(m_p->explain_xsp, doc, 0);
+
+            xmlFreeDoc(doc);
+            doc = rec_res;
+        }
+        if (!doc)
+        {
+            *error = YAZ_BIB1_UNSPECIFIED_ERROR;
+            *addinfo = odr_strdup(odr, "IR-Explain--1 problem. " 
+                                  "Could not obtain Torus records for Explain");
+        }
+        else
+        {
+            xmlNode *ptr = xmlDocGetRootElement(doc);
+            int hits = 0;
+            
+            xml_node_search(ptr, &hits, 0);
+
+            Z_APDU *apdu_res = odr.create_searchResponse(apdu_req, 0, 0);
+            apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
+            package.response() = apdu_res;
+            m_backend = b;
+        }
+        if (b->explain_doc)
+            xmlFreeDoc(b->explain_doc);
+        b->explain_doc = doc;
+        return m_backend;
+    }
+    else
+    {
+        *error = YAZ_BIB1_QUERY_TYPE_UNSUPP;
+        *addinfo = odr_strdup(odr, "IR-Explain---1 only supports CQL");
+        return m_backend;
+    }
+}
+
 void yf::Zoom::Frontend::handle_search(mp::Package &package)
 {
     Z_GDU *gdu = package.request().get();
@@ -1427,7 +1662,6 @@ void yf::Zoom::Frontend::handle_search(mp::Package &package)
         package.response() = apdu_res;
         return;
     }
-
     int proxy_step = 0;
 
 next_proxy:
@@ -1451,6 +1685,8 @@ next_proxy:
         package.response() = apdu_res;
         return;
     }
+    if (!b || b->enable_explain)
+        return;
 
     b->set_option("setname", "default");
 
@@ -1615,12 +1851,16 @@ next_proxy:
             switch (cerror)
             {
             case CCL_ERR_UNKNOWN_QUAL:
-                error = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
-                break;
             case CCL_ERR_TRUNC_NOT_LEFT: 
             case CCL_ERR_TRUNC_NOT_RIGHT:
             case CCL_ERR_TRUNC_NOT_BOTH:
-                error = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
+#ifdef CCL_ERR_TRUNC_NOT_EMBED
+            case CCL_ERR_TRUNC_NOT_EMBED:
+#endif
+#ifdef CCL_ERR_TRUNC_NOT_SINGLE
+            case CCL_ERR_TRUNC_NOT_SINGLE:
+#endif
+                error = YAZ_BIB1_UNSUPP_SEARCH;
                 break;
             }
             log_diagnostic(package, error, addinfo);
@@ -1751,19 +1991,42 @@ void yf::Zoom::Frontend::handle_present(mp::Package &package)
     Odr_int number_of_records_returned = 0;
     int error = 0;
     char *addinfo = 0;
-    Z_Records *records = get_records(package,
-        *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
-        &error, &addinfo, &number_of_records_returned, odr, m_backend,
-        pr->preferredRecordSyntax, element_set_name);
 
-    apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
-    if (records)
+    if (m_backend->enable_explain)
     {
-        apdu_res->u.presentResponse->records = records;
-        apdu_res->u.presentResponse->numberOfRecordsReturned =
-            odr_intdup(odr, number_of_records_returned);
+        Z_Records *records =
+            get_explain_records(
+                package,
+                *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
+                &error, &addinfo, &number_of_records_returned, odr, m_backend,
+                pr->preferredRecordSyntax, element_set_name);
+        
+        apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
+        if (records)
+        {
+            apdu_res->u.presentResponse->records = records;
+            apdu_res->u.presentResponse->numberOfRecordsReturned =
+                odr_intdup(odr, number_of_records_returned);
+        }
+        package.response() = apdu_res;
+    }
+    else
+    {
+        Z_Records *records =
+            get_records(package,
+                        *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
+                        &error, &addinfo, &number_of_records_returned, odr, m_backend,
+                        pr->preferredRecordSyntax, element_set_name);
+        
+        apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
+        if (records)
+        {
+            apdu_res->u.presentResponse->records = records;
+            apdu_res->u.presentResponse->numberOfRecordsReturned =
+                odr_intdup(odr, number_of_records_returned);
+        }
+        package.response() = apdu_res;
     }
-    package.response() = apdu_res;
 }
 
 void yf::Zoom::Frontend::handle_package(mp::Package &package)