From: Wolfram Schneider Date: Wed, 14 Apr 2010 11:58:10 +0000 (+0000) Subject: Merge branch 'nigiri' X-Git-Tag: CPAN-v1.02~54^2~93 X-Git-Url: http://git.indexdata.com/?p=irspy-moved-to-github.git;a=commitdiff_plain;h=87eef33eee0a92bf11aa4d4fcc061526f9176a50;hp=8c4c18f34a60df3af7d02337a466aa08c1673ade Merge branch 'nigiri' Conflicts: zebra/README --- diff --git a/bin/irspy-rewrite-records.pl b/bin/irspy-rewrite-records.pl new file mode 100644 index 0000000..bcc58e7 --- /dev/null +++ b/bin/irspy-rewrite-records.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + +# perl -I ../lib irspy-rewrite-records.pl localhost:8018/IR-Explain---1 + +use strict; +use warnings; +use ZOOM::IRSpy; +use ZOOM::IRSpy::Utils qw(render_record); + +my($dbname) = @ARGV; +die "$0 no database name specified" if !defined $dbname; +my $spy = new ZOOM::IRSpy($dbname, "admin", "fruitbat"); +my $rs = $spy->{conn}->search(new ZOOM::Query::CQL("cql.allRecords=1")); +print STDERR "rewriting ", $rs->size(), " target records"; + +foreach my $i (1 .. $rs->size()) { + my $xml = render_record($rs, $i-1, "zeerex"); + my $rec = $spy->{libxml}->parse_string($xml)->documentElement(); + ZOOM::IRSpy::_rewrite_zeerex_record($spy->{conn}, $rec); + print STDERR "."; +} +print STDERR "\nDone\n"; diff --git a/bin/test-xml-update.pl b/bin/test-xml-update.pl index 8cc551c..ad7f086 100755 --- a/bin/test-xml-update.pl +++ b/bin/test-xml-update.pl @@ -10,7 +10,7 @@ use warnings; use Getopt::Std; use ZOOM; use ZOOM::IRSpy::Utils qw(irspy_xpath_context modify_xml_document); -use ZOOM::IRSpy; # For _really_rewrite_record() +use ZOOM::IRSpy; # For _rewrite_zeerex_record() # This array copied from ../web/htdocs/details/edit.mc my @fields = @@ -80,7 +80,7 @@ print("Document modified with $nchanges change", $nchanges == 1 ? "" : "s", ": ", join(", ", map { $_->[2] } @changedFields), "\n"); if ($opts{w}) { - ZOOM::IRSpy::_really_rewrite_record($conn, $xc->getContextNode(), $id); + ZOOM::IRSpy::_rewrite_zeerex_record($conn, $xc->getContextNode(), $id); print "Rewrote record '$id'\n"; } diff --git a/lib/ZOOM/IRSpy.pm b/lib/ZOOM/IRSpy.pm index 16625d0..3e93b47 100644 --- a/lib/ZOOM/IRSpy.pm +++ b/lib/ZOOM/IRSpy.pm @@ -18,7 +18,8 @@ use ZOOM::IRSpy::Connection; use ZOOM::IRSpy::Stats; use ZOOM::IRSpy::Utils qw(cql_target render_record irspy_xpath_context irspy_make_identifier - irspy_record2identifier); + irspy_record2identifier calc_reliability_stats + modify_xml_document); our @ISA = qw(); our $VERSION = '1.02'; @@ -270,7 +271,9 @@ sub _hash { sub _irspy_to_zeerex { my $this = shift(); - my($conn, $save_xml) = @_; + my($conn) = @_; + + my $save_xml = $ENV{IRSPY_SAVE_XML}; my $irspy_doc = $conn->record()->{zeerex}->ownerDocument; if ($save_xml) { @@ -294,26 +297,37 @@ sub _irspy_to_zeerex { } -sub _rewrite_record { +sub _rewrite_irspy_record { my $this = shift(); my($conn) = @_; $conn->log("irspy", "rewriting XML record"); - my $rec = $this->_irspy_to_zeerex($conn, $ENV{IRSPY_SAVE_XML}); + my $rec = $this->_irspy_to_zeerex($conn); # Since IRSpy can run for a long time between writes back to the # database, it's quite possible for the server to have closed the # connection as idle. So re-establish it if necessary. $this->{conn}->connect($conn->option("host")); - _really_rewrite_record($this->{conn}, $rec); + _rewrite_zeerex_record($this->{conn}, $rec); $conn->log("irspy", "rewrote XML record"); } -sub _really_rewrite_record { +my $_reliabilityField = { + reliability => [ reliability => 0, + "Calculated reliability of server", + "e:serverInfo/e:reliability" ], +}; + +sub _rewrite_zeerex_record { my($conn, $rec, $oldid) = @_; + # Add reliability score + my $xc = irspy_xpath_context($rec); + my($nok, $nall, $percent) = calc_reliability_stats($xc); + modify_xml_document($xc, $_reliabilityField, { reliability => $percent }); + my $p = $conn->package(); $p->option(action => "specialUpdate"); my $xml = $rec->toString(); @@ -323,7 +337,6 @@ sub _really_rewrite_record { # This is the expression in the ID-making stylesheet # ../../zebra/zeerex2id.xsl - my $xc = irspy_xpath_context($rec); my $id = irspy_record2identifier($xc); if (defined $oldid && $id ne $oldid) { warn "IDs differ (old='$oldid' new='$id')"; @@ -439,7 +452,7 @@ sub check { } if (!defined $nextaddr) { $conn->log("irspy", "has no more tests: removing"); - $this->_rewrite_record($conn); + $this->_rewrite_irspy_record($conn); $conn->option(rewrote_record => 1); my $newconn = $this->_next_connection(); if (!defined $newconn) { diff --git a/lib/ZOOM/IRSpy/Utils.pm b/lib/ZOOM/IRSpy/Utils.pm index b1d2331..dfe4e99 100644 --- a/lib/ZOOM/IRSpy/Utils.pm +++ b/lib/ZOOM/IRSpy/Utils.pm @@ -20,7 +20,9 @@ our @EXPORT_OK = qw(utf8param irspy_identifier2target modify_xml_document bib1_access_point - render_record); + render_record + calc_reliability_string + calc_reliability_stats); use XML::LibXML; use XML::LibXML::XPathContext; @@ -275,6 +277,11 @@ sub _irspy_identifier2target { } +# Modifies the XML document for which $xc is an XPath context by +# inserting or replacing the values specified in the hash %$data. The +# keys are fieldnames, which are looked up in the register +# $fieldsByKey to determine, among other things, what their XPath is. + sub modify_xml_document { my($xc, $fieldsByKey, $data) = @_; @@ -772,4 +779,26 @@ sub render_record { } +sub calc_reliability_string { + my($xc) = @_; + + my($nok, $nall, $percent) = calc_reliability_stats($xc); + return "[untested]" if $nall == 0; + return "$nok/$nall = " . $percent . "%"; +} + + +sub calc_reliability_stats { + my($xc) = @_; + + my @allpings = $xc->findnodes("i:status/i:probe"); + my $nall = @allpings; + return (0, 0, 0) if $nall == 0; + my @okpings = $xc->findnodes('i:status/i:probe[@ok = "1"]'); + my $nok = @okpings; + my $percent = int(100*$nok/$nall); + return ($nok, $nall, $percent); +} + + 1; diff --git a/web/conf/apache2.0/xeno-dev.conf b/web/conf/apache2.0/xeno-dev.conf new file mode 100644 index 0000000..ab274ec --- /dev/null +++ b/web/conf/apache2.0/xeno-dev.conf @@ -0,0 +1,33 @@ +# This is the configuration that I use on the development machine xeno + + + ServerName x.irspy.indexdata.com + + ErrorLog /var/log/apache2/irspy-error.log + CustomLog /var/log/apache2/irspy-access.log combined + DocumentRoot /usr/local/src/git/irspy/web/htdocs + + SetHandler perl-script + PerlHandler HTML::Mason::ApacheHandler + + + PerlSetVar MasonArgsMethod mod_perl + + PerlAddVar MasonCompRoot "private => /usr/local/src/git/irspy/web/htdocs" + PerlSetVar MasonDataDir /tmp/irspy-mason + # IRSpyLibDir is used only to find source for online documentation + PerlSetVar IRSpyLibDir /usr/local/src/git/irspy/lib + PerlSetEnv PERL5LIB /usr/local/src/git/irspy/lib + + # We need +Parent to make PerlSwitches -I work properly ... don't ask. + PerlOptions +Parent + PerlSwitches -I/usr/local/src/git/irspy/lib + + + AuthType Basic + AuthName "IRSpy Administration" + AuthUserFile /usr/local/src/git/irspy/web/conf/htpasswd + Require user admin + + + diff --git a/web/htdocs/chrome/layout.mc b/web/htdocs/chrome/layout.mc index b99cb80..b1f054b 100644 --- a/web/htdocs/chrome/layout.mc +++ b/web/htdocs/chrome/layout.mc @@ -12,7 +12,7 @@ use ZOOM::IRSpy::Utils qw(utf8param isodate xml_encode cql_target cql_quote irspy_xpath_context irspy_make_identifier irspy_record2identifier irspy_identifier2target modify_xml_document - bib1_access_point); + bib1_access_point calc_reliability_string); % $r->content_type("text/html; charset=utf-8"); % my $text = $m->scomp($component, %ARGS); diff --git a/web/htdocs/details/edit.mc b/web/htdocs/details/edit.mc index be35470..e0403a4 100644 --- a/web/htdocs/details/edit.mc +++ b/web/htdocs/details/edit.mc @@ -352,7 +352,7 @@ if ($update && @changedFields) { "e:metaInfo/e:dateModified" ] }, { dateModified => isodate(time()) }); die "Didn't set dateModified!" if !@x; - ZOOM::IRSpy::_really_rewrite_record($conn, $xc->getContextNode(), + ZOOM::IRSpy::_rewrite_zeerex_record($conn, $xc->getContextNode(), $op eq "edit" ? $id : undef); } diff --git a/web/htdocs/details/find.mc b/web/htdocs/details/find.mc index 024437a..9edbe8d 100644 --- a/web/htdocs/details/find.mc +++ b/web/htdocs/details/find.mc @@ -54,6 +54,10 @@ Database Name + + Reliability at least + + <%doc>   diff --git a/web/htdocs/details/found.mc b/web/htdocs/details/found.mc index 13143c6..0f44196 100644 --- a/web/htdocs/details/found.mc +++ b/web/htdocs/details/found.mc @@ -19,22 +19,6 @@ sub navlink { return $url; } -# Identical to the same-named function in full.mc -# So maybe this should go into IRSpy::Utils.pm? -# Name changed (append 2) to prevent inadvertent clashes in Mason namespace -# -sub calc_reliability2 { - my($xc) = @_; - - my @allpings = $xc->findnodes("i:status/i:probe"); - my $nall = @allpings; - return "[untested]" if $nall == 0; - my @okpings = $xc->findnodes('i:status/i:probe[@ok = "1"]'); - my $nok = @okpings; - return "$nok/$nall = " . int(100*$nok/$nall) . "%"; -} - - # Just make this once; forge the connection on first use our $conn = undef; @@ -131,7 +115,7 @@ print_navlink(\%params, $last < $n, "Next", $skip+$count); <%perl> my $xc = irspy_xpath_context($rs->record($i-1)); my $title = $xc->find("e:databaseInfo/e:title") || "[UNTITLED]"; -my $reliability = calc_reliability2($xc); +my $reliability = calc_reliability_string($xc); my $host = $xc->find("e:serverInfo/e:host"); my $port = $xc->find("e:serverInfo/e:port"); my $db = $xc->find("e:serverInfo/e:database"); diff --git a/web/htdocs/details/full.mc b/web/htdocs/details/full.mc index abf5c01..73836a1 100644 --- a/web/htdocs/details/full.mc +++ b/web/htdocs/details/full.mc @@ -41,7 +41,7 @@ if ($n == 0) { [ "Implementation ID" => "i:status/i:implementationId" ], [ "Implementation Name" => "i:status/i:implementationName" ], [ "Implementation Version" => "i:status/i:implementationVersion" ], - [ "Reliability/reliability" => \&calc_reliability, $xc ], + [ "Reliability/reliability" => \&calc_reliability_wrapper, $xc ], [ "Services" => \&calc_init_options, $xc ], [ "Bib-1 Use attributes" => \&calc_ap, $xc, "bib-1" ], [ "Dan-1 Use attributes" => \&calc_ap, $xc, "dan-1" ], @@ -95,15 +95,9 @@ if ($n == 0) { % } <%perl> -sub calc_reliability { +sub calc_reliability_wrapper { my($id, $xc) = @_; - - my @allpings = $xc->findnodes("i:status/i:probe"); - my $nall = @allpings; - return "[untested]" if $nall == 0; - my @okpings = $xc->findnodes('i:status/i:probe[@ok = "1"]'); - my $nok = @okpings; - return "$nok/$nall = " . int(100*$nok/$nall) . "%"; + return calc_reliability_string($xc); } sub calc_init_options { diff --git a/web/htdocs/details/upload.mc b/web/htdocs/details/upload.mc index 5a5aab3..3809292 100644 --- a/web/htdocs/details/upload.mc +++ b/web/htdocs/details/upload.mc @@ -51,7 +51,7 @@ my $id = irspy_record2identifier($xc); my $conn = new ZOOM::Connection("localhost:8018/IR-Explain---1", 0, user => "admin", password => "fruitbat", elementSetName => "zeerex"); -ZOOM::IRSpy::_really_rewrite_record($conn, $xc->getContextNode()); +ZOOM::IRSpy::_rewrite_zeerex_record($conn, $xc->getContextNode());

Upload OK. diff --git a/zebra/.gitignore b/zebra/.gitignore index 546f36a..3403f67 100644 --- a/zebra/.gitignore +++ b/zebra/.gitignore @@ -1,5 +1,7 @@ records-2007-04-18 records-2007-05-01 records-2008-09-16 +records-2010-04-06 terse.properties log +db diff --git a/zebra/README b/zebra/README index 77b87b9..2898e93 100644 --- a/zebra/README +++ b/zebra/README @@ -111,4 +111,3 @@ zebraidx-2.0 update record-2010-04-06 zebraidx-2.0 commit zebrasrv-2.0 -f yazserver.xml - diff --git a/zebra/dom-filter/README b/zebra/dom-filter/README new file mode 100644 index 0000000..72a916f --- /dev/null +++ b/zebra/dom-filter/README @@ -0,0 +1,3 @@ +These files are the residue of an abandoned port of IRSpy's Zebra +configuration from the Alvis filter to the DOM filter. We will +probably return to this at some point. diff --git a/zebra/dom-filter/domfilterconf.xml b/zebra/dom-filter/domfilterconf.xml new file mode 100644 index 0000000..456da1f --- /dev/null +++ b/zebra/dom-filter/domfilterconf.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/zebra/dom-filter/zebra.cfg b/zebra/dom-filter/zebra.cfg new file mode 100644 index 0000000..76c3946 --- /dev/null +++ b/zebra/dom-filter/zebra.cfg @@ -0,0 +1,31 @@ +# $Id: zebra.cfg,v 1.12 2006-12-19 12:53:16 sondberg Exp $ + +# Where to look for config files +profilePath: .:/usr/local/share/idzebra-2.0/tab:/usr/share/idzebra-2.0/tab + +# Where to look for loadable zebra modules. Both these path components +# are necessary, since the former is used when installing from a Debian +# package and the latter when installing from a CVS build. +modulePath: /usr/lib/idzebra-2.0/modules +#modulePath: /usr/local/lib/idzebra-2.0/modules + +# store records and record keys internally +storeData: 1 +storeKeys: 1 + +# Use the "dom" filter with config file "domfilterconf.xml" +recordtype.xml: dom.domfilterconf.xml + +database: IR-Explain---1 +#database: Default + +# where to put registers, and other var content, and how large they may be +register: db/register:100G +shadow: db/shadow:100G +lockdir: db/lock +keytmpdir: db/tmp + +# Permissions for update +perm.anonymous: ra +perm.admin: rw +passwd.c: htpasswd diff --git a/zebra/dom-filter/zeerex2index.xsl b/zebra/dom-filter/zeerex2index.xsl new file mode 100644 index 0000000..527407a --- /dev/null +++ b/zebra/dom-filter/zeerex2index.xsl @@ -0,0 +1,21 @@ + + + + + + + + + + + + + diff --git a/zebra/pqf.properties b/zebra/pqf.properties index a1d4715..045239e 100644 --- a/zebra/pqf.properties +++ b/zebra/pqf.properties @@ -26,6 +26,8 @@ index.net.method = 1=net:method index.net.host = 1=net:host 4=3 index.net.port = 1=net:port 4=3 index.net.path = 1=net:path 4=3 +index.zeerex.reliability = 1=zeerex:reliability 4=109 +index.zeerex.reliabilityAtLeast = 1=zeerex:reliability 2=4 4=109 index.dc.title = 1=dc:title index.dc.creator = 1=dc:creator index.dc.description = 1=dc:description diff --git a/zebra/records-2010-04-06.tar.gz b/zebra/records-2010-04-06.tar.gz new file mode 100644 index 0000000..ef5319d Binary files /dev/null and b/zebra/records-2010-04-06.tar.gz differ diff --git a/zebra/zeerex2index.xsl b/zebra/zeerex2index.xsl index 76618cf..7c4f5eb 100644 --- a/zebra/zeerex2index.xsl +++ b/zebra/zeerex2index.xsl @@ -71,6 +71,9 @@ + + +