-$Id: Changes,v 1.23 2007-09-18 16:58:01 mike Exp $
Revision history for Perl extension ZOOM::IRSpy.
bin/Makefile
bin/irspy-stats.pl
bin/irspy.pl
-bin/irspy_xsltproc.pl
+bin/irspy-xsltproc.pl
bin/reindex.pl
bin/setrlimit.c
lib/ZOOM/IRSpy.pm
web/htdocs/admin/delete.html
web/htdocs/admin/edit.html
web/htdocs/admin/index.html
-web/htdocs/admin/upload.html
+web/htdocs/upload.html
web/htdocs/ap.html
web/htdocs/beach.jpeg
web/htdocs/chrome/layout.mc
-# $Id: Makefile.PL,v 1.14 2007-05-09 12:04:36 mike Exp $
use 5.008;
use strict;
-
ZOOM::IRSpy
===========
you may need one of the following packages for development:
sudo apt-get install \
- make perl-doc libxml2-utils tidy
+ make perl-doc libxml2-utils tidy yaz libyaz4-dev
-and may enable mod_apreq
+and may enable mod_apreq: sudo a2enmod apreq
-sudo a2enmod apreq
+Note: mod_perl works only safe without threads. Please use the non-threaded
+debian package: sudo apt-get install apache2-mpm-prefork
To install this module type the following:
-$Id: Notes,v 1.1 2006-05-31 12:04:19 mike Exp $
Some things that it would be nice to check (varying priorities!)
-$Id: architecture-notes,v 1.1 2007-02-20 12:10:02 mike Exp $
The basic approach is this: The IRSpy::Connection is an IRSpy-specific
subclass of ZOOM::Connection, and its only important extra
-$Id: separate-apache,v 1.1 2007-03-22 12:25:36 mike Exp $
/etc/apache-irspy -- Apache configuration
/var/log/apache-irspy -- Apache logs
-# $Id: .gdbinit,v 1.7 2007-03-15 11:34:51 mike Exp $
set env YAZ_LOG=irspy,irspy_test,irspy_task,zoom,zoom_details,irspy_event
set args -I../lib irspy.pl -t Quick -f 'net.host=*indexdata*' localhost:8018/IR-Explain---1
set args -I../lib irspy.pl -n 2 -t Quick localhost:8018/IR-Explain---1 z3950.loc.gov:7090/Voyager bagel.indexdata.dk/gils bagel.indexdata.dk:210/marc
-# $Id: .perldb,v 1.1 2007-02-09 10:48:37 mike Exp $
print "=== loading .perldb ===\n"
#!/usr/bin/perl
-
+#
# perl -I ../lib irspy-rewrite-records.pl localhost:8018/IR-Explain---1
+use lib '../lib';
+use Data::Dumper;
+use Getopt::Long;
+use ZOOM::IRSpy;
+use ZOOM::IRSpy::Utils qw(render_record validate_record);
+
use strict;
use warnings;
-use ZOOM::IRSpy;
-use ZOOM::IRSpy::Utils qw(render_record);
-my($dbname) = @ARGV;
-die "$0 no database name specified" if !defined $dbname;
-my $spy = new ZOOM::IRSpy($dbname, "admin", "fruitbat");
-my $rs = $spy->{conn}->search(new ZOOM::Query::CQL("cql.allRecords=1"));
-print STDERR "rewriting ", $rs->size(), " target records";
+my $irspy_to_zeerex_xsl = '../xsl/irspy2zeerex.xsl';
+my $debug = 1;
+my $cql_query = "cql.allRecords=1";
-foreach my $i (1 .. $rs->size()) {
- my $xml = render_record($rs, $i-1, "zeerex");
+sub usage {
+ my $message = shift;
+
+ warn "$message\n" if defined $message;
+
+ <<EOF
+usage $0 [ options ] database
+
+--xslt=$irspy_to_zeerex_xsl set xslt sheet
+--debug=0..2 verbose level
+--query=$cql_query
+EOF
+}
+
+GetOptions(
+ "xslt" => \$irspy_to_zeerex_xsl,
+ "debug=i" => \$debug,
+ "query=s" => \$cql_query,
+);
+
+my $dbname = shift;
+die usage("no database name specified\n") if !defined $dbname;
+
+$ZOOM::IRSpy::irspy_to_zeerex_xsl = $irspy_to_zeerex_xsl
+ if $irspy_to_zeerex_xsl;
+
+my $spy = new ZOOM::IRSpy( $dbname, "admin", "fruitbat" );
+my $rs = $spy->{conn}->search( new ZOOM::Query::CQL($cql_query) );
+print STDERR "rewriting ", $rs->size(), " target records\n" if $debug;
+
+foreach my $i ( 1 .. $rs->size() ) {
+ my $xml = render_record( $rs, $i - 1, "zeerex" );
my $rec = $spy->{libxml}->parse_string($xml)->documentElement();
- ZOOM::IRSpy::_rewrite_zeerex_record($spy->{conn}, $rec);
- print STDERR ".";
+
+ if ( $debug >= 2 ) {
+ my ( $ok, $errors ) = validate_record($rec);
+ if ( !$ok ) {
+ my @e = @$errors;
+ my $id = shift @e;
+ print "Id: $id => ", join( " / ", @e ), "\n";
+ }
+ }
+ ZOOM::IRSpy::_rewrite_zeerex_record( $spy->{conn}, $rec );
+ print STDERR "." if $debug == 1;
}
-print STDERR "\nDone\n";
+print STDERR "Done\n" if $debug;
+
home=/usr/local/src/git
cd $home/irspy/bin || exit 2
logdir=../tmp
+lockfile=../tmp/irspy-update.lock
+: ${irspy_test=Main}
+if [ -f $lockfile ]; then
+ pid=`cat $lockfile`
+ if kill -0 $pid 2>/dev/null; then
+ echo "This script is already running with pid: $pid"
+ exit 1
+ fi
+fi
+echo $$ > $lockfile || exit 2
+
+weekday=`date '+%w'`
for i in 0 1 2 3 4 5 6
do
- logfile=$logdir/irspy-mod-$i.log.`date '+%w'`
- YAZ_LOG=irspy,irspy_test nice -10 time perl -I../lib irspy.pl -n 50 -d -M 3500 -a -t Main -m 7,$i localhost:8018/IR-Explain---1 > $logfile 2>&1
+ logfile=$logdir/irspy-mod-$i.log.$weekday
+ YAZ_LOG=irspy,irspy_test,irspy_task nice -10 time perl -I../lib irspy.pl -n 50 -d -M 3500 -f'cql.allRecords=1 not zeerex.disabled = 1' -t $irspy_test -m 7,$i localhost:8018/IR-Explain---1 > $logfile 2>&1
+
+ sleep 1 # catch ctr-c before compressing the log
gzip -f $logfile
done
+rm -f $lockfile
+
-# $Id: IRSpy.pm,v 1.90 2008-07-16 11:42:13 mike Exp $
package ZOOM::IRSpy;
our $irspy_to_zeerex_xsl = dirname(__FILE__) . '/../../xsl/irspy2zeerex.xsl';
our $debug = 0;
our $xslt_max_depth = 250;
+our $max_timeout_errors = 3;
# Enumeration for callback functions to return
while (1) {
my @copy_conn = @conn; # avoid alias problems after splice()
my $nconn = scalar(@copy_conn);
+
foreach my $i0 (0 .. $#copy_conn) {
my $conn = $copy_conn[$i0];
#print "connection $i0 of $nconn/", scalar(@conn), " is $conn\n";
next if !defined $conn;
+
if (!$conn->current_task()) {
if (!$conn->next_task()) {
# Out of tasks: we need a new test
"checking for next test after '$address'");
$nextaddr = $this->_next_test($address);
}
+
+ if (ZOOM::IRSpy::Test::zoom_error_timeout_check($conn)) {
+ $conn->log("irspy", "Got to many timeouts, stop testing");
+ undef $nextaddr;
+ }
+
if (!defined $nextaddr) {
$conn->log("irspy", "has no more tests: removing");
$this->_rewrite_irspy_record($conn);
my $task = $conn->next_task();
die "no next task queued for $conn" if !defined $task;
+
+ # do not run the next task if we got too many timeouts
+ if (ZOOM::IRSpy::Test::zoom_error_timeout_check($conn)) {
+ $conn->log("irspy_task", "Got to many timeouts for this target, do not start a new task");
+ next;
+ }
+
$conn->log("irspy_task", "preparing task $task");
$conn->next_task(0);
$conn->current_task($task);
-# $Id: Connection.pm,v 1.21 2007-12-20 12:35:01 mike Exp $
package ZOOM::IRSpy::Connection;
-# $Id: Maintenance.pod,v 1.4 2006-09-18 10:27:57 mike Exp $
package ZOOM::IRSpy::Maintenance;
-# $Id: Node.pm,v 1.6 2007-02-28 17:34:54 mike Exp $
package ZOOM::IRSpy::Node;
-# $Id: Record.pm,v 1.28 2007-12-12 08:49:58 mike Exp $
package ZOOM::IRSpy::Record;
### I don't think there's any reason for this to be separate from
target => $target,
parser => $parser,
zeerex => $parser->parse_string($zeerex)->documentElement(),
+ zoom_error => { TIMEOUT => 0 },
}, $class;
#Scalar::Util::weaken($this->{irspy});
return $this;
}
+sub zoom_error { return shift->{'zoom_error'} }
sub _empty_zeerex_record {
my($target) = @_;
-# $Id: Stats.pm,v 1.7 2007-10-31 16:42:13 mike Exp $
package ZOOM::IRSpy::Stats;
# Record syntax support by database
foreach my $node ($xc->findnodes('e:recordInfo/e:recordSyntax/@name')) {
- $this->{recordSyntaxes}->{$node->findvalue(".")}++;
+ $this->{recordSyntaxes}->{lc($node->findvalue("."))}++;
}
# Explain support
# Top Domains
my $host = $xc->findvalue('e:serverInfo/e:host');
$host =~ s/.*\.//;
- $this->{domains}->{$host}++;
+ $this->{domains}->{lc($host)}++;
# Implementation
foreach my $node ($xc->findnodes('i:status/i:serverImplementationName/@value')) {
-# $Id: Task.pm,v 1.7 2007-08-01 15:11:03 mike Exp $
package ZOOM::IRSpy::Task;
-# $Id: Connect.pm,v 1.6 2006-11-02 16:11:44 mike Exp $
# See ZOOM/IRSpy/Task/Search.pm for documentation
-# $Id: Retrieve.pm,v 1.6 2007-05-09 11:05:30 mike Exp $
package ZOOM::IRSpy::Task::Retrieve;
-# $Id: Search.pm,v 1.16 2007-12-18 11:59:42 mike Exp $
package ZOOM::IRSpy::Task::Search;
-# $Id: Test.pm,v 1.8 2007-04-18 15:24:45 mike Exp $
package ZOOM::IRSpy::Test;
use Scalar::Util;
+use Exporter 'import';
+our @EXPORT = qw(zoom_error_timeout_update zoom_error_timeout_check);
+
=head1 NAME
ZOOM::IRSpy::Test - base class for tests in IRSpy
=cut
+
sub subtests { () }
sub timeout { undef }
}
+our $max_timeout_errors = $ZOOM::IRSpy::max_timeout_errors;
+
+sub zoom_error_timeout_update {
+ my ($conn, $exception) = @_;
+
+ if ($exception =~ /Timeout/i) {
+ $conn->record->zoom_error->{TIMEOUT}++;
+ $conn->log("irspy_test", "Increase timeout error counter to: " .
+ $conn->record->zoom_error->{TIMEOUT});
+ }
+}
+
+sub zoom_error_timeout_check {
+ my $conn = shift;
+
+ if ($conn->record->zoom_error->{TIMEOUT} >= $max_timeout_errors) {
+ $conn->log("irspy_test", "Got $max_timeout_errors or more timeouts, give up...");
+ return 1;
+ }
+
+ return 0;
+}
+
=head1 SEE ALSO
ZOOM::IRSpy
-# $Id: Main.pm,v 1.14 2007-04-18 15:24:55 mike Exp $
package ZOOM::IRSpy::Test::Main;
-# $Id: Ping.pm,v 1.28 2008-07-16 11:51:09 mike Exp $
# See the "Main" test package for documentation
-# $Id: Quick.pm,v 1.4 2007-04-18 15:25:07 mike Exp $
package ZOOM::IRSpy::Test::Quick;
-# $Id: Fetch.pm,v 1.31 2009-04-21 13:42:58 mike Exp $
# See the "Main" test package for documentation
use ZOOM::IRSpy::Test;
our @ISA = qw(ZOOM::IRSpy::Test);
+our $max_timeout_errors = $ZOOM::IRSpy::max_timeout_errors;
# These queries
my @queries = (
$conn->log("irspy_test", "Fetch test search (", $task->render_query(), ") ",
ref $event && $event->isa("ZOOM::Exception") ?
"failed: $event" : "found $n records (event=$event)");
+
+ # remember how often a target record hit a timeout
+ if (ref $event && $event->isa("ZOOM::Exception")) {
+ if ($event =~ /Timeout/i) {
+ $conn->record->zoom_error->{TIMEOUT}++;
+ $conn->log("irspy_test", "Increase timeout error counter to: " .
+ $conn->record->zoom_error->{TIMEOUT});
+ }
+ }
+
if ($n == 0) {
$task->{rs}->destroy();
my $qindex = $udata->{queryindex}+1;
my $q = $queries[$qindex];
return ZOOM::IRSpy::Status::TEST_SKIPPED
- if !defined $q;
+ if !defined $q || $conn->record->zoom_error->{TIMEOUT} >= $max_timeout_errors;
$conn->log("irspy_test", "Trying another search ...");
$conn->irspy_search_pqf($queries[$qindex], { queryindex => $qindex }, {},
'librismarc',
'mab',
'normarc',
-# 'opac',
+ 'opac',
'picamarc',
'rusmarc',
'summary',
-# $Id: Main.pm,v 1.1 2006-10-23 13:54:52 sondberg Exp $
package ZOOM::IRSpy::Test::Record::Main;
-# $Id: Main.pm,v 1.1 2006-11-02 11:46:40 sondberg Exp $
package ZOOM::IRSpy::Test::ResultSet::Main;
-# $Id: Named.pm,v 1.6 2007-08-01 15:11:20 mike Exp $
# See the "Main" test package for documentation
my($conn, $task, $test_args, $exception) = @_;
$conn->log("irspy_test", "Named resultset check failed:", $exception);
+ zoom_error_timeout_update($conn, $exception);
return ZOOM::IRSpy::Status::TASK_DONE;
}
-# $Id: Bib1.pm,v 1.18 2007-03-15 11:40:27 mike Exp $
# See the "Main" test package for documentation
$conn->log("irspy_test", "search on access-point $attr had error: ",
$exception);
update($conn, $attr, 0);
+ zoom_error_timeout_update($conn, $exception);
return ZOOM::IRSpy::Status::TEST_BAD
if ($exception->code() == 1 || # permanent system error
'ok' => $ok);
}
-
1;
-# $Id: Boolean.pm,v 1.5 2007-03-15 11:40:39 mike Exp $
# See the "Main" test package for documentation
$conn->log("irspy_test", "search using boolean operator ", $operator,
" had error: ", $exception);
update($conn, $operator, 0);
+ zoom_error_timeout_update($conn, $exception);
return ZOOM::IRSpy::Status::TASK_DONE;
}
-# $Id: CQL.pm,v 1.1 2007-05-01 15:32:16 mike Exp $
# See the "Main" test package for documentation
$task->{rs}->destroy();
$conn->log("irspy_test", "CQL search on '$index' had error: $exception");
$conn->record()->store_result("search_cql", index => $index, ok => 0);
+ zoom_error_timeout_update($conn, $exception);
return ZOOM::IRSpy::Status::TEST_BAD
if $exception->code() == 11; # Unsupported query type
-# $Id: DBDate.pm,v 1.4 2007-02-23 15:03:44 mike Exp $
# This plugin tests searching on BIB-1 access-point 1011 (Date/time
# added to db), the significance of which is that this search
my $rec = $conn->record();
$rec->append_entry("irspy:status", "<irspy:search_dbdate ok='0'>" .
isodate(time()) . "</irspy:search_dbdate>");
+ zoom_error_timeout_update($conn, $exception);
return ZOOM::IRSpy::Status::TEST_BAD;
}
-# $Id: Dan1.pm,v 1.7 2007-03-15 11:40:52 mike Exp $
# See the "Main" test package for documentation
$conn->log("irspy_test", "search on access-point $attr had error: ",
$exception);
update($conn, $attr, 0);
+ zoom_error_timeout_update($conn, $exception);
### How about TEST_BAD if $exception->code() == 121?
return ZOOM::IRSpy::Status::TASK_DONE;
-# $Id: Explain.pm,v 1.7 2007-03-15 11:41:24 mike Exp $
# See the "Main" test package for documentation
$task->{rs}->destroy();
$conn->log("irspy_test", "Explain category lookup failed: ", $exception);
update($conn, $category, 0);
+ zoom_error_timeout_update($conn, $exception);
return ZOOM::IRSpy::Status::TEST_BAD
if ($exception->code() == 109 || # Database unavailable
-# $Id: Main.pm,v 1.8 2007-05-01 15:30:38 mike Exp $
package ZOOM::IRSpy::Test::Search::Main;
-# $Id: Title.pm,v 1.10 2007-02-23 15:03:44 mike Exp $
# See the "Main" test package for documentation
my $rec = $conn->record();
$rec->append_entry("irspy:status", "<irspy:search_title ok='0'>" .
isodate(time()) . "</irspy:search_title>");
+ zoom_error_timeout_update($conn, $exception);
return ZOOM::IRSpy::Status::TEST_BAD;
}
-# $Id: Utils.pm,v 1.38 2009-04-15 18:16:45 wosch Exp $
package ZOOM::IRSpy::Utils;
use Exporter 'import';
our @EXPORT_OK = qw(utf8param
+ trimField
+ utf8paramTrim
isodate
xml_encode
cql_quote
modify_xml_document
bib1_access_point
render_record
+ validate_record
calc_reliability_string
calc_reliability_stats);
return $cooked;
}
-
# Utility functions follow, exported for use of web UI
sub utf8param_apache1 {
my($r, $key, $value) = @_;
$year+1900, $mon+1, $mday, $hour, $min, $sec);
}
+# strips whitespaces at start and ends of a field
+sub trimField {
+ my $field = shift;
+
+ $field =~ s/^\s+//;
+ $field =~ s/\s+$//;
+
+ return $field;
+}
+
+# utf8param() with trim
+sub utf8paramTrim {
+ my $result = utf8param(@_);
+
+ if (defined $result) {
+ $result = trimField($result);
+ }
+
+ return $result;
+}
# I can't -- just can't, can't, can't -- believe that this function
# isn't provided by one of the core XML modules. But the evidence all
}
} else {
- next if !$value; # No need to create a new empty node
+ next if !defined $value; # No need to create a new empty node
my($ppath, $selector) = $xpath =~ /(.*)\/(.*)/;
dom_add_node($xc, $ppath, $selector, $value, @addAfter);
#print "New $key ($xpath) = '$value'<br/>\n";
return (0, 0, 0) if $nall == 0;
my @okpings = $xc->findnodes('i:status/i:probe[@ok = "1"]');
my $nok = @okpings;
- my $percent = int(100*$nok/$nall);
+ my $percent = int(100*$nok/$nall + 0.5);
return ($nok, $nall, $percent);
}
+#
+# validate_record( record, ( "port" => 1, "database" => 1, "country" => 0, ... ))
+#
+sub validate_record {
+ my $rec = shift;
+ my %args = @_;
+
+ my %required = map { $_ => 1 } qw/port host database protocol/;
+ my %optional = map { $_ => 1 } qw/country type hosturl contact language/;
+ my %tests = ( %required, %args );
+
+ my $xc = irspy_xpath_context($rec);
+
+ my $protocol = $xc->findnodes("e:serverInfo/\@protocol") || "";
+ my $port = $xc->findnodes("e:serverInfo/e:port") || "";
+ my $host = $xc->findnodes("e:serverInfo/e:host") || "";
+ my $dbname = $xc->findnodes("e:serverInfo/e:database") || "";
+
+ my $id = irspy_make_identifier($protocol, $host, $port, $dbname);
+
+ if ($protocol =~ /\s+$/ || $dbname =~ /\s+$/) {
+ warn "xxx: $protocol:$host:$port:$dbname: whitespaces\n";
+ }
+
+ my @errors = $id;
+
+ if ($tests{'protocol'}) {
+ push(@errors, 'protocol number is not valid') if $protocol !~ /^(z39\.50|sru|srw|tcp)$/i;
+ }
+
+ if ($tests{'port'}) {
+ push(@errors, 'port number is not valid') if $port !~ /^\d+$/;
+ }
+
+ if ($tests{'host'}) {
+ push(@errors, 'host name is not valid') if $host !~ /^[0-9a-z]+[0-9a-z\.\-]*\.[0-9a-z]+$/i;
+ }
+
+ if ($tests{'database'}) {
+ push(@errors, 'database name is not valid') if $dbname =~ m,/,i;
+ push(@errors, 'database has trailing spaces') if $dbname =~ /^\s+|\s+$/;
+ }
+
+ if ($tests{'hosturl'}) {
+ my $hosturl = $xc->findnodes("i:status/i:hostURL") || "";
+ push(@errors, 'This hosturl name is not valid') if $hosturl !~ /^\w+$/i;
+ }
+
+ return ( !$#errors, \@errors );
+}
1;
-# $Id: Web.pm,v 1.5 2007-05-08 16:20:04 mike Exp $
package ZOOM::IRSpy::Web;
-# $Id: WebService.pod,v 1.5 2007-01-24 09:28:02 mike Exp $
package ZOOM::IRSpy::WebService;
#!/usr/bin/perl -w
-# $Id: x.pl,v 1.1 2007-02-28 16:52:34 mike Exp $
### This should be massaged into a test-suite script in ../../../t
-# $Id: Pod.pm,v 1.22 2006-10-06 11:33:07 mike Exp $
package ZOOM::Pod;
-# $Id: Simple.pm,v 1.2 2006-07-21 11:29:17 mike Exp $
package ZOOM::XML::Simple;
ErrorLog /var/log/apache2/irspy-test-error.log
CustomLog /var/log/apache2/irspy-test-access.log combined
DocumentRoot /usr/local/src/git/irspy-test/web/htdocs
+
+ Alias /robots.txt /usr/local/src/git/irspy-test/web/htdocs/robots-test.txt
+
<FilesMatch "\.(html|css)$">
SetHandler perl-script
PerlHandler HTML::Mason::ApacheHandler
-# $Id: cfspy.conf,v 1.4 2009-04-16 18:31:20 wosch Exp $
-#
# Sample configuration for running an IRSpy web-site under Apache 2.x.
#
# This is the configuration that I use on the development machine
+++ /dev/null
-%# $Id: add.html,v 1.3 2007-07-16 11:55:31 mike Exp $
-<& /chrome/layout.mc, %ARGS,
- title => "Add target record",
- component => "/details/edit.mc", op => "new" &>
--- /dev/null
+<& /chrome/layout.mc, %ARGS,
+ title => "Add target record",
+ component => "/details/edit.mc", op => "new" &>
-%# $Id: all.html,v 1.1 2007-02-20 19:17:14 mike Exp $
<& /chrome/layout.mc, %ARGS, title => "Test all targets",
component => "/details/check.mc", id => "" &>
-%# $Id: check.html,v 1.1 2007-02-20 19:17:14 mike Exp $
<%args>
@id
$test => undef
-%# $Id: delete.html,v 1.1 2007-02-20 19:17:14 mike Exp $
<%args>
$id => undef
</%args>
-%# $Id: edit.html,v 1.1 2007-02-20 19:17:14 mike Exp $
<%args>
$op
$id => undef
-%# $Id: ap.html,v 1.1 2007-06-28 12:19:28 mike Exp $
<%args>
$set
</%args>
-%# $Id: layout.mc,v 1.38 2007-07-17 13:07:57 mike Exp $
<%args>
$debug => undef
$title
use URI::Escape qw(uri_escape uri_escape_utf8);
use ZOOM;
use ZOOM::IRSpy::Web;
-use ZOOM::IRSpy::Utils qw(utf8param isodate xml_encode cql_target cql_quote
+use ZOOM::IRSpy::Utils qw(utf8param trimField utf8paramTrim isodate xml_encode cql_target cql_quote
irspy_xpath_context irspy_make_identifier
irspy_record2identifier
irspy_identifier2target modify_xml_document
<td valign="top" class="panel1">
<p>
<a href="/"><b>Home</b></a><br/>
- <a href="/admin/all.html">Test all targets</a><br/>
+ <!-- <a href="/admin/all.html">Test all targets</a><br/> -->
<a href="/find.html">Find a target</a><br/>
- <a href="/add.html">Add a target</a><br/>
+ <a href="/add_target.html">Add a target</a><br/>
<a href="/upload.html">Upload a target</a><br/>
<a href="/stats.html">Statistics</a><br/>
</p>
-%# $Id: ap.mc,v 1.1 2007-06-28 12:19:37 mike Exp $
<%args>
$id
$set
-%# $Id: check.mc,v 1.18 2007-05-09 10:45:14 mike Exp $
<%args>
@id
$test => "Quick"
-%# $Id: country-list.mc,v 1.2 2007-03-30 11:14:56 mike Exp $
<%perl>
return [
"",
-%# $Id: delete.mc,v 1.7 2007-06-27 11:09:32 mike Exp $
<%args>
$id
$really => 0
-%# $Id: doc.mc,v 1.4 2007-05-03 14:13:19 mike Exp $
<%once>
use Pod::Html;
use IO::Dir;
-%# $Id: edit.mc,v 1.40 2009-04-15 18:16:46 wosch Exp $
<%args>
$op
$id => undef ### should be extracted using utf8param()
user => "admin", password => "fruitbat",
elementSetName => "zeerex");
-my $protocol = utf8param($r, "protocol");
-my $host = utf8param($r, "host");
-my $port = utf8param($r, "port");
-my $dbname = utf8param($r, "dbname");
+my $protocol = utf8paramTrim($r, "protocol");
+my $host = utf8paramTrim($r, "host");
+my $port = utf8paramTrim($r, "port");
+my $dbname = utf8paramTrim($r, "dbname");
+my $title = utf8paramTrim($r, "title");
if ((!defined $port || $port eq "") &&
(defined $protocol && $protocol ne "")) {
if (defined $protocol && $protocol ne "" &&
defined $host && $host ne "" &&
defined $port && $port ne "" &&
+ defined $title && $title ne "" &&
defined $dbname && $dbname ne "") {
$newid = irspy_make_identifier($protocol, $host, $port, $dbname);
}
} elsif (!defined $newid) {
# Tried to create new record but data is insufficient
print qq[<p class="error">
- Please specify protocol, host, port and database name.</p>\n];
+ Please specify title, protocol, host, port and database name.</p>\n];
undef $update;
} elsif ($host !~ /^\w+\.[\w.]*\w$/i) {
print qq[<p class="error">
qw(e:title e:description) ],
[ subjects => 2, "Subjects", "e:databaseInfo/e:subjects",
qw(e:title e:description) ],
+ [ disabled => [ qw(0 1) ],
+ "Target Test Disabled", "i:status/i:disabled" ],
);
# Update record with submitted data
my %data;
foreach my $key (&utf8param($r)) {
next if grep { $key eq $_ } qw(op id update);
- $data{$key} = utf8param($r, $key);
+ $data{$key} = trimField( utf8param($r, $key) );
}
my @changedFields = modify_xml_document($xc, \%fieldsByKey, \%data);
if ($update && @changedFields) {
-%# $Id: error.mc,v 1.1 2006-09-19 11:11:44 mike Exp $
<%args>
$message
</%args>
-%# $Id: find.mc,v 1.12 2009-04-15 18:16:46 wosch Exp $
% if (&utf8param($r,"_search")) {
% $m->comp("found.mc");
% } else {
-%# $Id: found.mc,v 1.33 2009-04-15 18:16:46 wosch Exp $
<%once>
sub print_navlink {
my($params, $cond, $caption, $skip) = @_;
-%# $Id: full.mc,v 1.30 2007-07-03 13:10:50 mike Exp $
<%args>
$id
</%args>
-%# $Id: home.mc,v 1.11 2008-07-03 08:14:21 mike Exp $
<div class="essay">
<h2>The Z39.50 Target Directory</h2>
<p>
-%# $Id: libtype-list.mc,v 1.2 2007-03-30 11:14:56 mike Exp $
<%perl>
return [
"", qw(Academic Public Corporate Special National Education Other)
-%# $Id: software.mc,v 1.3 2007-04-18 12:13:01 mike Exp $
<div class="essay">
<p>
Index Data uses
-%# $Id: stats.mc,v 1.9 2007-11-02 12:49:28 mike Exp $
<%doc>
Here are the headings in the Z-Spy version:
The ten most commonly supported Bib-1 Use attributes
<p>Recalculating stats</p>
% }
<& table, stats => $stats, data => "bib1AccessPoints",
- title => "The ten most commonly supported Bib-1 Use attributes",
- headings => [ "Attribute", "Name"],
+ title => "The twenty most commonly supported Bib-1 Use attributes",
+ headings => [ "Attribute", "Name"], maxrows => 20,
col3 => sub { bib1_access_point(@_) } &>
<& table, stats => $stats, data => "recordSyntaxes",
title => "Record syntax support by database",
- headings => [ "Record Syntax"] &>
+ headings => [ "Record Syntax"], maxrows => 30 &>
<& table, stats => $stats, data => "explain",
title => "Explain Support",
headings => [ "Explain Category"] &>
my $hr;
$hr = $stats->{$data};
my @sorted = sort { $hr->{$b} <=> $hr->{$a} || $a <=> $b } keys %$hr;
-my $n = @sorted; $n = $maxrows if @sorted > 10;
+my $n = @sorted; $n = $maxrows if @sorted > 10 && $n > $maxrows;
foreach my $i (1..$n) {
my $key = $sorted[$i-1];
</%perl>
-%# $Id: upload.mc,v 1.3 2009-04-16 18:09:44 wosch Exp $
<%args>
$filename => undef
</%args>
-%# $Id: doc.html,v 1.1 2006-09-28 16:48:00 mike Exp $
<& /chrome/layout.mc, %ARGS, component => "/details/doc.mc",
title => defined $ARGS{module} ? $ARGS{module} : "Documentation" &>
-%# $Id: find.html,v 1.2 2006-09-19 16:30:25 mike Exp $
<& /chrome/layout.mc, %ARGS, title => "Find a target",
component => "/details/find.mc" &>
-%# $Id: full.html,v 1.1 2006-10-20 16:57:40 mike Exp $
<& /chrome/layout.mc, %ARGS, title => "Full target record",
component => "/details/full.mc" &>
-%# $Id: help.html,v 1.5 2006-12-06 14:07:17 mike Exp $
<%args>
$help
</%args>
-%# $Id: edit.html,v 1.1 2006-12-05 12:26:37 mike Exp $
<p class="small">
(The fields used to describe targets in the IRSpy database are for
the most part those described in the
-%# $Id: author.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
The author element should contain the name of the person or
organisation to be credited with the creation of the database, and
-%# $Id: contact.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
The contact element is used to record information on a contact
person for the database. This should include at least a name and
-%# $Id: country.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
The country in which the service is hosted, if known.
</p>
-%# $Id: dbname.html,v 1.1 2006-12-05 12:26:37 mike Exp $
<p>
For Z39.50 databases, contains the name of the Z39.50 database
which the service provides. If the service provides multiple
-%# $Id: description.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
Should contain a description of why this database might be of
--- /dev/null
+ <p>The target will not be tested automatically if set and is not zero.</p>
+
+ <p>This option is mainly for the administrator of IRSpy in case of trouble.</p>
-%# $Id: extent.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
Used to describe the completeness of the database, or the range of
material that is included in it. For example a database which
-%# $Id: history.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
Any information which is considered useful regarding the history of
the database may be recorded in the history element. This might
-%# $Id: host.html,v 1.2 2006-12-06 14:20:25 mike Exp $
<p>
The address of the server which hosts the service. This address
should be in a name which will resolve to the correct IP
-%# $Id: hosturl.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
A URL for the organisation which hosts the database. This URL may
or not refer to the same host as the
-%# $Id: language.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
Used to record the languages used in the database records (as
opposed to the target record). For example, in an English-language
-%# $Id: password.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<& username.html &>
-%# $Id: port.html,v 1.1 2006-12-05 12:26:37 mike Exp $
<p>
The port that the service is running on. For Z39.50 servers, this
is often 210, 3950 or 7090. IRSpy needs this information in order
-%# $Id: protocol.html,v 1.1 2006-12-05 12:26:37 mike Exp $
<p>
Used to record the protocol that should be used to connect to the
server. The default value is
-%# $Id: restrictions.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
If there are any restrictions on the usage or availability of the
database or its contents then these should be recorded in this
-%# $Id: subjects.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
If the database concerns particular subjects from a controlled
vocabulary then these may be recorded using this field. These
-%# $Id: title.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
The name by which the database is known.
</p>
-%# $Id: type.html,v 1.1 2006-12-05 16:40:46 mike Exp $
<p>
The type of library that this is a catlogue for. This field is not
part of the ZeeRex data model, and may require further thought -
-%# $Id: username.html,v 1.1 2006-12-06 14:22:22 mike Exp $
<p>
Some Z39.50 servers require authentication before access is
allowed. For such servers, a suitable username and password may be
-%# $Id: info.html,v 1.1 2007-04-26 13:56:46 mike Exp $
<!-- No generic help needed for "info" category -->
-%# $Id: reliability.html,v 1.1 2007-04-26 13:56:53 mike Exp $
<p>
Measures the reliability of the target only in its ability to
respond to connections: the display indicates the number of
-%# $Id: link.mc,v 1.9 2007-06-25 10:38:35 mike Exp $
<%args>
$help
</%args>
-%# $Id: index.html,v 1.3 2006-09-15 16:47:47 mike Exp $
<& /chrome/layout.mc, %ARGS, title => "Welcome",
component => "/details/home.mc" &>
-%# $Id: sru-auth.html,v 1.1 2007-08-21 09:48:41 mike Exp $
<%doc>
In order to test the authentication feature of Simple2ZOOM, we
need an authenticator script, accessible via HTTP. The
-%# $Id: raw.html,v 1.9 2007-05-11 13:53:40 mike Exp $
<%args>
$id
</%args>
--- /dev/null
+User-agent: *
+Disallow: /
User-agent: *
-Disallow: /full.html
-Disallow: /all.html
+Disallow: /admin/
Disallow: /add.html
-Disallow: /upload.html
+Disallow: /add_target.html
+Disallow: /all.html
+Disallow: /doc.html
+Disallow: /find.html
+Disallow: /full.html
Disallow: /raw.html
-Disallow: /admin/
+Disallow: /stats.html
+Disallow: /upload.html
-%# $Id: software.html,v 1.1 2007-03-30 11:43:59 mike Exp $
<& /chrome/layout.mc, %ARGS, title => "About the IRSpy software",
component => "/details/software.mc" &>
-%# $Id: stats.html,v 1.1 2006-12-14 16:11:03 mike Exp $
<& /chrome/layout.mc, %ARGS, title => "Target statistic",
component => "/details/stats.mc" &>
-/* $Id: style.css,v 1.4 2007-03-22 07:57:34 mike Exp $ */
body {
color: black;
background: white;
-%# $Id: upload.html,v 1.1 2007-07-16 11:55:20 mike Exp $
<& /chrome/layout.mc, %ARGS,
title => "Upload target record",
component => "/details/upload.mc" &>
-# $Id: Makefile,v 1.1 2006-12-05 12:26:37 mike Exp $
-
PNG = ../htdocs/help-20px.png ../htdocs/help-16px.png
all: $(PNG)
<?xml version="1.0"?>
<!--
- $Id: irspy2zeerex.xsl,v 1.21 2007-06-28 14:01:56 sondberg Exp $
This stylesheet is used by IRSpy to map the internal mixed Zeerex/IRSpy
record format into the Zeerex record which we store.
<?xml version="1.0" encoding="UTF-8"?>
<!--
- $Id: use-attr-names.xml,v 1.2 2006-10-27 12:50:06 sondberg Exp $
-
Maps numeric use attributes to human readable names - credits to Per
-->
<mapping>
ZEBRA_DIR= db
ZEBRAIDX= zebraidx-2.0
ZEBRAIDX_TEST= zebraidx-2.0 -c zebra-test.cfg
+IRSPY_DATABASE= $${IRSpyDbName-"localhost:8018/IR-Explain---1"}
# debugging
ZEBRA_TEST_DIR= db-test
rm -rf records-${DATE}.old
-test -e records-${DATE} && mv records-${DATE} records-${DATE}.old
mkdir records-${DATE}
- cd records-${DATE} && ../../bin/irspy-dump.pl localhost:8018/IR-Explain---1
+ cd records-${DATE} && ../../bin/irspy-dump.pl ${IRSPY_DATABASE}
cd records-${DATE} && ( for i in *.xml; do tidy -xml -i -m -w 140 $$i; done ) > /dev/null 2>&1
tar cf - records-${DATE} | gzip > records-${DATE}.tar.gz
-$Id: README,v 1.21 2007-05-09 16:48:31 mike Exp $
What's what in this directory:
zebraidx-2.0 commit
zebrasrv-2.0 -f yazserver.xml
+
-# $Id: crontab,v 1.4 2007-09-18 17:02:19 mike Exp $
+# Example crontab file for a weekly update cycle, testing 1/7 of the
+# database every day. See also the script ../bin/irspy-update.sh
+#
# m h dom mon dow command
-0 1 * * 0 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,0 localhost:8018/IR-Explain---1
-0 1 * * 1 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,1 localhost:8018/IR-Explain---1
-0 1 * * 2 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,2 localhost:8018/IR-Explain---1
-0 1 * * 3 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,3 localhost:8018/IR-Explain---1
-0 1 * * 4 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,4 localhost:8018/IR-Explain---1
-0 1 * * 5 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,5 localhost:8018/IR-Explain---1
-0 1 * * 6 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,6 localhost:8018/IR-Explain---1
+0 1 * * * cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,`date '+%w'` localhost:8018/IR-Explain---1
-# $Id: zebra.cfg,v 1.12 2006-12-19 12:53:16 sondberg Exp $
# Where to look for config files
profilePath: .:/usr/local/share/idzebra-2.0/tab:/usr/share/idzebra-2.0/tab
#! /usr/bin/perl -w
-# $Id: ezeerex2pqfproperties.pl,v 1.7 2006-06-20 11:24:22 mike Exp $
#
# Run like this:
# ./ezeerex2pqfproperties.pl zeerex.xml
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: filterconf.xml,v 1.4 2007-05-11 08:30:23 mike Exp $ -->
<schemaInfo>
<schema name="index" stylesheet="zeerex2index.xsl"
identifier="http://indexdata.dk/zebra/xslt/1"/>
<?xml version="1.0" encoding="ISO-8859-1"?>
-<!-- $Id: form.html,v 1.3 2007-01-24 09:28:03 mike Exp $ -->
<!DOCTYPE html
PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-# $Id: pqf.properties,v 1.18 2007-03-29 17:14:13 mike Exp $
#
# Properties file to drive the YAZ CQL-to-PQF converter for Zebra.
# This specifies the interpretation of various CQL indexes, relations,
# Extensions
index.zeerex.libType = 1=zeerex:libType 4=3
index.zeerex.country = 1=zeerex:country 4=3
+index.zeerex.disabled = 1=zeerex:disabled 4=3
# Relation attributes are selected according to the CQL relation by
# looking up the "relation.<relation>" property:
-$Id: profile,v 1.2 2006-06-16 14:16:12 mike Exp $
Notes on the indexes in the ZeeRex profile, with indications of
whether they are yet supported by the Zebra configuration in this
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: yazserver.xml,v 1.4 2007-01-24 09:28:03 mike Exp $ -->
<yazgfs>
<listen id="tcp8019">tcp:@:8019</listen>
<server id="alvis" listenref="tcp8019">
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: yazserver.xml,v 1.4 2007-01-24 09:28:03 mike Exp $ -->
<yazgfs>
<listen id="tcp8018">tcp:@:8018</listen>
<server id="alvis" listenref="tcp8018">
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: zeerex.xml,v 1.15 2007-01-24 09:28:03 mike Exp $ -->
<explain xmlns="http://explain.z3950.org/dtd/2.0/">
<serverInfo protocol="SRW/SRU/Z39.50" version="1.1" method="GET/POST">
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: zeerex.xml,v 1.15 2007-01-24 09:28:03 mike Exp $ -->
<explain xmlns="http://explain.z3950.org/dtd/2.0/">
<serverInfo protocol="SRW/SRU/Z39.50" version="1.1" method="GET/POST">
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: zeerex2dc.xsl,v 1.2 2007-04-27 14:04:40 mike Exp $ -->
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:z="http://indexdata.dk/zebra/xslt/1"
xmlns:e="http://explain.z3950.org/dtd/2.0/"
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: zeerex2id.xsl,v 1.2 2007-04-27 14:04:40 mike Exp $ -->
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:e="http://explain.z3950.org/dtd/2.0/"
version="1.0">
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: zeerex2index.xsl,v 1.15 2009-04-16 15:28:26 mike Exp $ -->
<!-- See the ZeeRex profile at http://srw.cheshire3.org/profiles/ZeeRex/ -->
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:z="http://indexdata.dk/zebra/xslt/1"
</xsl:for-each>
<!-- recordInfo -->
- <z:index name="zeerex:recordSyntax" type="0">
+ <z:index name="zeerex:recordSyntax" type="w">
<xsl:value-of select="e:recordInfo/e:recordSyntax/@name"/>
<!-- ### But @identifier is an OID for Z39.50 -->
</z:index>
<z:index name="zeerex:country" type="0">
<xsl:value-of select="i:status/i:country"/>
</z:index>
+ <z:index name="zeerex:disabled" type="0">
+ <xsl:value-of select="i:status/i:disabled"/>
+ </z:index>
</z:record>
</xsl:template>
<?xml version="1.0"?>
-<!--
- $Id: zeerex2noauth.xsl,v 1.2 2007-06-25 12:32:13 sondberg Exp $
--->
<xsl:stylesheet
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:ex="http://explain.z3950.org/dtd/2.0/"
<?xml version="1.0" encoding="UTF-8"?>
-<!-- $Id: zeerex2zeerex.xsl,v 1.1 2006-04-13 14:53:18 mike Exp $ -->
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
<xsl:output indent="yes" method="xml" version="1.0" encoding="UTF-8"/>
<xsl:template match="/">
projects / irspy-moved-to-github.git / commitdiff