From: Wolfram Schneider Date: Wed, 14 Apr 2010 11:58:10 +0000 (+0000) Subject: Merge branch 'nigiri' X-Git-Tag: CPAN-v1.02~54^2~93 X-Git-Url: http://git.indexdata.com/?p=irspy-moved-to-github.git;a=commitdiff_plain;h=87eef33eee0a92bf11aa4d4fcc061526f9176a50;hp=f01e52b20eaac746a863f8a7b9102c6bf2e22f5a Merge branch 'nigiri' Conflicts: zebra/README --- diff --git a/README b/README index 362353e..e19d4c1 100644 --- a/README +++ b/README @@ -1,4 +1,3 @@ -$Id: README,v 1.6 2007-04-18 15:22:48 mike Exp $ ZOOM::IRSpy =========== @@ -9,6 +8,37 @@ protocols. It is a successor to the ZSpy program. INSTALLATION +for debian/ubuntu: +------------------- + +DEBIAN DEPENDENCIES + +If you have a Debian-based operating system, you can get what you need +by installing the following packages from the operating system's +package store, using apt-get: + +sudo apt-get install \ + libnet-z3950-zoom-perl \ + libxml-libxml-common-perl \ + liburi-perl \ + libxml-libxml-perl \ + libapache2-mod-perl2 \ + libhtml-mason-perl \ + libapache2-request-perl \ + libapache2-mod-apreq2 \ + libxml-libxslt-perl \ + idzebra-2.0 \ + +you may need one of the following packages for development: + +sudo apt-get install \ + make perl-doc libxml2-utils tidy + +and may enable mod_apreq + +sudo a2enmod apreq + + To install this module type the following: perl Makefile.PL @@ -21,11 +51,11 @@ DEPENDENCIES This module requires these other modules and libraries, in addition to those specified in Makefile.PL: - Zebra, release 1.4.0 or later: see http://indexdata.com/zebra/ + Zebra, release 2.0 or later: see http://www.indexdata.com/zebra/ COPYRIGHT AND LICENCE -Copyright (C) 2006 by Index Data ApS. +Copyright (C) 2006-2010 by Index Data ApS. This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself, either Perl version 5.8.7 or, diff --git a/archive/.cvsignore b/archive/.cvsignore deleted file mode 100644 index e641f3e..0000000 --- a/archive/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -zspy.zip -zeerez-record-display diff --git a/archive/.gitignore b/archive/.gitignore new file mode 100644 index 0000000..e641f3e --- /dev/null +++ b/archive/.gitignore @@ -0,0 +1,2 @@ +zspy.zip +zeerez-record-display diff --git a/bin/.cvsignore b/bin/.cvsignore deleted file mode 100644 index b5f7560..0000000 --- a/bin/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -test-zoom-c -setrlimit diff --git a/bin/.gitignore b/bin/.gitignore new file mode 100644 index 0000000..b5f7560 --- /dev/null +++ b/bin/.gitignore @@ -0,0 +1,2 @@ +test-zoom-c +setrlimit diff --git a/bin/irspy-dump.pl b/bin/irspy-dump.pl index a87177b..4e62a8b 100755 --- a/bin/irspy-dump.pl +++ b/bin/irspy-dump.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -w +#!/usr/bin/perl # Invoke as: # $ mkdir records-2010-04-06 @@ -6,6 +6,8 @@ # $ irspy-dump.pl irspy.indexdata.com:8018/IR-Explain---1 # $ cd .. # $ tar cfz records-2010-04-06.tar.gz records-2010-04-06 +# +# or: cd zebra; make dump use strict; use warnings; @@ -23,14 +25,17 @@ my $rs = $conn->search_pqf('@attr 1=_ALLRECORDS @attr 2=103 ""'); my $n = $rs->size(); $| = 1; print "$0: dumping $n records\n"; + +my $step = 200; +my $step2 = 10; foreach my $i (1..$n) { - print "."; - print " $i/$n (", int($i*100/$n), "%)\n" if $i % 50 == 0; + print "." if $i % $step2 == 0; + print " $i/$n (", int($i*100/$n), "%)\n" if $i % $step == 0; my $rec = $rs->record($i-1); my $xml = $rec->render(); open F, ">$i.xml"; print F $xml; close F; } -print " $n/$n (100%)\n" if $n % 50 != 0; +print " $n/$n (100%)\n" if $n % $step != 0; print "complete\n"; diff --git a/bin/irspy.pl b/bin/irspy.pl index 3a8918f..963c1b5 100755 --- a/bin/irspy.pl +++ b/bin/irspy.pl @@ -1,7 +1,5 @@ -#!/usr/bin/perl -w +#!/usr/bin/perl -# $Id: irspy.pl,v 1.29 2007-09-18 16:58:18 mike Exp $ -# # Run like this: # YAZ_LOG=irspy,irspy_test IRSPY_SAVE_XML=1 perl -I../lib irspy.pl -t Quick localhost:8018/IR-Explain---1 Z39.50:amicus.oszk.hu:1616/ANY # YAZ_LOG=irspy,irspy_test sudo ./setrlimit -n 3000 -u mike -- perl -I../lib irspy.pl -t Main -a localhost:8018/IR-Explain---1 @@ -18,31 +16,33 @@ # I have no idea why this directory is not in Ubuntu's default Perl # path, but we need it because just occasionally overload.pm:88 # requires Scalar::Util, which is in this directory. +#use lib '/usr/share/perl/5.8.7'; -use lib '/usr/share/perl/5.8.7'; use Scalar::Util; - -use strict; -use warnings; use Getopt::Std; use ZOOM::IRSpy::Web; use Carp; +use strict; +use warnings; + $SIG{__DIE__} = sub { my($msg) = @_; confess($msg); }; my %opts; -if (!getopts('wt:af:n:m:', \%opts) || @ARGV < 1) { +if (!getopts('dwt:af:n:m:M:', \%opts) || @ARGV < 1) { print STDERR "\ Usage $0: [options] [ ...] + -d debug -w Use ZOOM::IRSpy::Web subclass -t Run the specified [default: all tests] -a Test all targets (slow!) -f Test targets found by the specified query -n Number of connection to keep in active set -m , Only test targets whose hash mod is + -M max_depth maximum number of nested template calls and variables/params "; exit 1; } @@ -51,6 +51,15 @@ my($dbname, @targets) = @ARGV; my $class = "ZOOM::IRSpy"; $class .= "::Web" if $opts{w}; +if ($opts{M} && $opts{M} > 0) { + no warnings; + $ZOOM::IRSpy::xslt_max_depth = $opts{M}; +} +if ($opts{d}) { + no warnings; + $ZOOM::IRSpy::debug = $opts{d}; +} + my $spy = $class->new($dbname, "admin", "fruitbat", $opts{n}); if (@targets) { $spy->targets(@targets); diff --git a/bin/irspy.sh b/bin/irspy.sh new file mode 100755 index 0000000..fea2758 --- /dev/null +++ b/bin/irspy.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# +# wrapper for irspy.pl +# +# run irspy with a smaller set of records in a loop to avoid out-of-memory +# + +home=/usr/local/src/git +cd $home/irspy/bin || exit 2 +logdir=../tmp + +for i in 0 1 2 3 4 5 6 +do + logfile=$logdir/irspy-mod-$i.log.`date '%w'` + YAZ_LOG=irspy,irspy_test nice -10 time perl -I../lib irspy.pl -n 50 -d -M 3500 -a -t Main -m 7,$i localhost:8018/IR-Explain---1 > $logfile 2>&1 + gzip -f $logfile +done + diff --git a/bin/irspy_xsltproc.pl b/bin/irspy_xsltproc.pl index 03e641d..8f37204 100755 --- a/bin/irspy_xsltproc.pl +++ b/bin/irspy_xsltproc.pl @@ -1,28 +1,55 @@ -#!/usr/bin/perl -w +#!/usr/bin/perl -# $Id: irspy_xsltproc.pl,v 1.4 2007-06-28 13:57:53 sondberg Exp $ -# ------------------------------------------------------------------ # This script is only for debugging purposes - it takes a raw IRspy # xml output document as argument and executes the irspy2zeerex.xsl # transformation right in front of you: # -# ./irspy_xsltproc.pl irspy_output_raw.xml -# +# ./irspy_xsltproc.pl irspy_output_raw.xml ... -use strict; -use warnings; +use Getopt::Long; +use Data::Dumper; use lib '../lib'; use ZOOM::IRSpy; -if (@ARGV && $ARGV[0] eq "-d") { - shift; - XML::LibXSLT->debug_callback(\&xslt_debug); +use strict; + +#use warnings; + +sub usage { + + < \$xslt_debug, + "v" => \$verbose, + "f=s" => \$irspy_to_zeerex_xsl, +); + +die usage if $#ARGV < 0; +XML::LibXSLT->debug_callback( \&xslt_debug ) if defined $xslt_debug; + +$ZOOM::IRSpy::irspy_to_zeerex_xsl = $irspy_to_zeerex_xsl + if $irspy_to_zeerex_xsl; + my $dbname = 'localhost:8018/IR-Explain---1'; -my $spy = new ZOOM::IRSpy($dbname, "admin", "fruitbat"); -my $source_file = shift || die("$0: Please specify xml instance file"); -my $source_doc = $spy->{libxml}->parse_file($source_file); -my $results = $spy->{irspy_to_zeerex_style}->transform($source_doc); +my $spy = new ZOOM::IRSpy( $dbname, "admin", "fruitbat" ); + +warn Dumper($spy) if $verbose; +foreach my $source_file (@ARGV) { + my $source_doc = $spy->{libxml}->parse_file($source_file); + my $results = $spy->{irspy_to_zeerex_style}->transform($source_doc); + + print $results->toString(1); +} -print $results->toString(1); diff --git a/lib/ZOOM/IRSpy.pm b/lib/ZOOM/IRSpy.pm index b1d07c7..3e93b47 100644 --- a/lib/ZOOM/IRSpy.pm +++ b/lib/ZOOM/IRSpy.pm @@ -22,8 +22,10 @@ use ZOOM::IRSpy::Utils qw(cql_target render_record modify_xml_document); our @ISA = qw(); -our $VERSION = '1.01'; +our $VERSION = '1.02'; our $irspy_to_zeerex_xsl = dirname(__FILE__) . '/../../xsl/irspy2zeerex.xsl'; +our $debug = 0; +our $xslt_max_depth = 250; # Enumeration for callback functions to return @@ -69,6 +71,7 @@ sub new { my $class = shift(); my($dbname, $user, $password, $activeSetSize) = @_; + my @options; push @options, (user => $user, password => $password) if defined $user; @@ -78,10 +81,15 @@ sub new { my $xslt = new XML::LibXSLT; + # raise the maximum number of nested template calls and variables/params (default 250) + warn "raise the maximum number of nested template calls: $xslt_max_depth\n" if $debug; + $xslt->max_depth($xslt_max_depth); + $xslt->register_function($ZOOM::IRSpy::Utils::IRSPY_NS, 'strcmp', \&ZOOM::IRSpy::Utils::xslt_strcmp); my $libxml = new XML::LibXML; + warn "use irspy_to_zeerex_xsl xslt sheet: $irspy_to_zeerex_xsl\n" if $debug; my $xsl_doc = $libxml->parse_file($irspy_to_zeerex_xsl); my $irspy_to_zeerex_style = $xslt->parse_stylesheet($xsl_doc); diff --git a/lib/ZOOM/IRSpy/Node.pm b/lib/ZOOM/IRSpy/Node.pm index f912b94..9d2f5c7 100644 --- a/lib/ZOOM/IRSpy/Node.pm +++ b/lib/ZOOM/IRSpy/Node.pm @@ -6,6 +6,7 @@ use 5.008; use strict; use warnings; +use Scalar::Util; =head1 NAME @@ -57,13 +58,15 @@ changed. sub new { my $class = shift(); my($name, @subnodes) = @_; - return bless { + my $this = bless { name => $name, subnodes => \@subnodes, address => undef, # filled in by resolve() previous => undef, # filled in by resolve() next => undef, # filled in by resolve() }, $class; + + return $this; } =head2 name() diff --git a/lib/ZOOM/IRSpy/Record.pm b/lib/ZOOM/IRSpy/Record.pm index 7444d70..e80fe12 100644 --- a/lib/ZOOM/IRSpy/Record.pm +++ b/lib/ZOOM/IRSpy/Record.pm @@ -8,6 +8,7 @@ use 5.008; use strict; use warnings; +use Scalar::Util; use XML::LibXML; use XML::LibXML::XPathContext; use ZOOM::IRSpy::Utils qw(xml_encode isodate irspy_xpath_context); @@ -36,12 +37,17 @@ sub new { ### Parser should be in the IRSpy object my $parser = new XML::LibXML(); - return bless { + my $this = bless { irspy => $irspy, target => $target, parser => $parser, zeerex => $parser->parse_string($zeerex)->documentElement(), }, $class; + + #Scalar::Util::weaken($this->{irspy}); + #Scalar::Util::weaken($this->{parser}); + + return $this; } diff --git a/lib/ZOOM/IRSpy/Stats.pm b/lib/ZOOM/IRSpy/Stats.pm index af667dc..f701740 100644 --- a/lib/ZOOM/IRSpy/Stats.pm +++ b/lib/ZOOM/IRSpy/Stats.pm @@ -5,6 +5,8 @@ package ZOOM::IRSpy::Stats; use 5.008; use strict; use warnings; + +use Scalar::Util; use ZOOM::IRSpy::Utils qw(irspy_xpath_context); =head1 NAME diff --git a/lib/ZOOM/IRSpy/Task.pm b/lib/ZOOM/IRSpy/Task.pm index 8e3f251..4340198 100644 --- a/lib/ZOOM/IRSpy/Task.pm +++ b/lib/ZOOM/IRSpy/Task.pm @@ -6,6 +6,8 @@ use 5.008; use strict; use warnings; +use Scalar::Util; + =head1 NAME ZOOM::IRSpy::Task - base class for tasks in IRSpy @@ -34,7 +36,7 @@ sub new { my $class = shift(); my($conn, $udata, $options, %cb) = @_; - return bless { + my $this = bless { irspy => $conn->{irspy}, conn => $conn, udata => $udata, @@ -42,6 +44,11 @@ sub new { cb => \%cb, timeRegistered => time(), }, $class; + + #Scalar::Util::weaken($this->{irspy}); + #Scalar::Util::weaken($this->{udata}); + + return $this; } diff --git a/lib/ZOOM/IRSpy/Test.pm b/lib/ZOOM/IRSpy/Test.pm index e73061a..765cd4f 100644 --- a/lib/ZOOM/IRSpy/Test.pm +++ b/lib/ZOOM/IRSpy/Test.pm @@ -6,6 +6,8 @@ use 5.008; use strict; use warnings; +use Scalar::Util; + =head1 NAME ZOOM::IRSpy::Test - base class for tests in IRSpy diff --git a/lib/ZOOM/IRSpy/Utils.pm b/lib/ZOOM/IRSpy/Utils.pm index dcf33ae..dfe4e99 100644 --- a/lib/ZOOM/IRSpy/Utils.pm +++ b/lib/ZOOM/IRSpy/Utils.pm @@ -6,6 +6,8 @@ use 5.008; use strict; use warnings; +use Scalar::Util; + use Exporter 'import'; our @EXPORT_OK = qw(utf8param isodate @@ -262,7 +264,7 @@ sub _irspy_identifier2target { if !defined $id; my($protocol, $target) = ($id =~ /(.*?):(.*)/); - if (uc($protocol) eq "Z39.50") { + if (uc($protocol) eq "Z39.50" || uc($protocol) eq "TCP") { return "tcp:$target"; } elsif (uc($protocol) eq "SRU") { return "sru=get,http:$target"; diff --git a/t/ZOOM-IRSpy.t b/t/ZOOM-IRSpy.t index b60bbe0..e9bd612 100644 --- a/t/ZOOM-IRSpy.t +++ b/t/ZOOM-IRSpy.t @@ -1,9 +1,38 @@ -# $Id: ZOOM-IRSpy.t,v 1.1 2007-04-18 11:23:01 mike Exp $ - # change 'tests => 1' to 'tests => last_test_to_print'; -use Test::More tests => 1; -BEGIN { use_ok('ZOOM::IRSpy') }; + +use Test::More qw(no_plan); + +BEGIN { + use_ok('ZOOM::IRSpy'); +} # Insert your test code below, the Test::More module is use()ed here so read # its man page ( perldoc Test::More ) for help writing this test script. +use strict; +use warnings; + +my $irspy_to_zeerex_xsl = 'xsl/irspy2zeerex.xsl'; + +$ZOOM::IRSpy::irspy_to_zeerex_xsl = $irspy_to_zeerex_xsl + if $irspy_to_zeerex_xsl; + +my $dbname = 'localhost:8018/IR-Explain---1'; +my $spy = new ZOOM::IRSpy( $dbname, "admin", "fruitbat" ); + +isa_ok( $spy, 'ZOOM::IRSpy' ); + +# test for failure if template not exists +eval { + $ZOOM::IRSpy::irspy_to_zeerex_xsl = '/nonexist'; + $ZOOM::IRSpy::debug = 1; + $ZOOM::IRSpy::xslt_max_depth = 100; + $spy = new ZOOM::IRSpy( $dbname, "admin", "fruitbat" ); +}; + +like( $@, qr/No such file or directory/, "xslt configure test" ); + +1; + +__DATA__; +foo diff --git a/web/conf/README b/web/conf/README index 9761991..50cb780 100644 --- a/web/conf/README +++ b/web/conf/README @@ -1,3 +1,9 @@ +!!!Note: most part of this document is outdated!!! + +Please have a look at the README in the top-level directory. + +April 2010, Wolfram + $Id: README,v 1.5 2006-10-31 14:17:54 sondberg Exp $ To install the web UI (assuming you're using some version of Apache as diff --git a/web/conf/apache1.3/newshawarma-proxy.conf b/web/conf/apache1.3/newshawarma-proxy.conf deleted file mode 100644 index 1df7db1..0000000 --- a/web/conf/apache1.3/newshawarma-proxy.conf +++ /dev/null @@ -1,16 +0,0 @@ -# $Id: newshawarma-proxy.conf,v 1.3 2007-06-25 10:39:22 mike Exp $ -# -# A very simple configuration to proxy the IRSpy server to a separate -# Apache instance running on the same host. This separation is useful -# to isolate IRSpy from other sites' errors and vice versa. - - - ServerName irspy.indexdata.com - ServerAlias targettest.indexdata.com - ProxyPass / http://irspy.indexdata.com:81/ - ProxyPassReverse / http://irspy.indexdata.com:81/ - - # These are the logs for the proxying operation - ErrorLog /var/log/apache/irspy-error.log - CustomLog /var/log/apache/irspy-access.log combined - diff --git a/web/conf/apache1.3/newshawarma.conf b/web/conf/apache1.3/newshawarma.conf deleted file mode 100644 index 1e0e5e6..0000000 --- a/web/conf/apache1.3/newshawarma.conf +++ /dev/null @@ -1,45 +0,0 @@ -# $Id: newshawarma.conf,v 1.4 2009-05-26 07:29:56 mike Exp $ -# -# Sample configuration for running an IRSpy web-site under Apache 1.3. -# -# This is the configuration that I use on the live machine, -# newshawarma, which is a PC running Debian GNU/Linux 3.1. -# On this machine, the IRSpy software is at -# /home/mike/cvs/irspy -# -# This file can be included in the Apache configuration by adding a -# line like this to apache.conf or one of its included files: -# Include /home/mike/cvs/irspy/web/conf/apache1.3/newshawarma.conf -# -# When running on Debian, don't forget to -# install libapache-request-perl as well as libhtml-mason-perl, -# otherwise Mason will fail mysteriously. -# -# And do NOT attempt to install ZAP on the same apache, that is known -# to fail! - - - ServerName irspy.indexdata.com - ErrorLog /var/log/apache-irspy/irspy-error.log - CustomLog /var/log/apache-irspy/irspy-access.log combined - DocumentRoot /home/mike/cvs/irspy/web/htdocs - - SetHandler perl-script - PerlHandler HTML::Mason::ApacheHandler - - PerlAddVar MasonCompRoot "private => /home/mike/cvs/irspy/web/htdocs" - PerlSetVar MasonDataDir /home/mike/cvs/irspy/web/data - PerlSetVar IRSpyLibDir /home/mike/cvs/irspy/lib - PerlSetEnv PERL5LIB /home/mike/cvs/irspy/lib - - AuthType Basic - AuthName "IRSpy Administration" - AuthUserFile /home/mike/cvs/irspy/web/conf/htpasswd - Require user admin - - - - - ServerName targettest.indexdata.com - Redirect permanent / http://irspy.indexdata.com/ - diff --git a/web/conf/apache1.3/rafa.conf b/web/conf/apache1.3/rafa.conf deleted file mode 100644 index 5b44dfb..0000000 --- a/web/conf/apache1.3/rafa.conf +++ /dev/null @@ -1,40 +0,0 @@ -# $Id: rafa.conf,v 1.1 2007-05-02 13:54:40 mike Exp $ -# -# Sample configuration for running an IRSpy web-site under Apache 1.3. -# -# This is the configuration that I use on the development machine, -# xeno, which is a Fujitu-Siemens SCENIC X102 PC running Ubuntu Linux -# release 6.06. On this machine, the IRSpy software is at -# /usr/local/src/cvs/irspy -# -# This file can be included in the Apache configuration by adding a -# line like this to apache.conf or one of its included files: -# Include /usr/local/src/cvs/irspy/web/conf/apache1.3/xeno.conf -# -# When running on Ubuntu (and probably Debian), don't forget to -# install libapache-request-perl as well as libhtml-mason-perl, -# otherwise Mason will fail mysteriously. - - - ServerName x.irspy.indexdata.com - ErrorLog /var/log/apache/irspy-error.log - CustomLog /var/log/apache/irspy-access.log combined - DocumentRoot /usr/local/src/cvs/irspy/web/htdocs - - SetHandler perl-script - PerlHandler HTML::Mason::ApacheHandler - - PerlAddVar MasonCompRoot "private => /usr/local/src/cvs/irspy/web/htdocs" - PerlSetVar MasonDataDir /usr/local/src/cvs/irspy/web/data - PerlSetVar IRSpyLibDir /usr/local/src/cvs/irspy/lib - PerlSetEnv PERL5LIB /usr/local/src/cvs/irspy/lib - PerlInitHandler Apache::Reload - PerlSetVar ReloadAll Off - PerlSetVar ReloadModules "ZOOM::IRSpy::Utils" - - AuthType Basic - AuthName "IRSpy Administration" - AuthUserFile /usr/local/src/cvs/irspy/web/conf/htpasswd - Require user admin - - diff --git a/web/conf/apache1.3/test.conf b/web/conf/apache1.3/test.conf deleted file mode 100644 index 255bd3e..0000000 --- a/web/conf/apache1.3/test.conf +++ /dev/null @@ -1,37 +0,0 @@ -# $Id: test.conf,v 1.5 2007-02-20 18:51:37 mike Exp $ -# -# Sample configuration for running an IRSpy web-site under Apache 1.3. -# -# This is the configuration that I use on the staging machine, -# test, which is a PC running Debian GNU/Linux 3.1. -# On this machine, the IRSpy software is at -# /usr/local/src/cvs/irspy -# -# This file can be included in the Apache configuration by adding a -# line like this to apache.conf or one of its included files: -# Include /usr/local/src/cvs/irspy/web/conf/apache1.3/test.conf -# -# When running on Debian, don't forget to -# install libapache-request-perl as well as libhtml-mason-perl, -# otherwise Mason will fail mysteriously. - - - ServerName irspy.indexdata.com - ErrorLog /var/log/apache/irspy-error.log - CustomLog /var/log/apache/irspy-access.log combined - DocumentRoot /usr/local/src/cvs/irspy/web/htdocs - - SetHandler perl-script - PerlHandler HTML::Mason::ApacheHandler - - PerlAddVar MasonCompRoot "private => /usr/local/src/cvs/irspy/web/htdocs" - PerlSetVar MasonDataDir /usr/local/src/cvs/irspy/web/data - PerlSetVar IRSpyLibDir /usr/local/src/cvs/irspy/lib - PerlSetEnv PERL5LIB /usr/local/src/cvs/irspy/lib - - AuthType Basic - AuthName "IRSpy Administration" - AuthUserFile /usr/local/src/cvs/irspy/web/conf/htpasswd - Require user admin - - diff --git a/web/conf/apache1.3/xeno.conf b/web/conf/apache1.3/xeno.conf deleted file mode 100644 index 7391a84..0000000 --- a/web/conf/apache1.3/xeno.conf +++ /dev/null @@ -1,39 +0,0 @@ -# $Id: xeno.conf,v 1.9 2008-08-29 18:58:13 mike Exp $ -# -# Sample configuration for running an IRSpy web-site under Apache 1.3. -# -# This is the configuration that I use on the development machine, -# xeno, which is a Fujitu-Siemens SCENIC X102 PC running Ubuntu Linux -# release 6.06. On this machine, the IRSpy software is at -# /usr/local/src/cvs/irspy -# -# This file can be included in the Apache configuration by adding a -# line like this to apache.conf or one of its included files: -# Include /usr/local/src/cvs/irspy/web/conf/apache1.3/xeno.conf -# -# When running on Ubuntu (and probably Debian), don't forget to -# install libapache-request-perl as well as libhtml-mason-perl, -# otherwise Mason will fail mysteriously. - -NameVirtualHost 192.168.1.75 - - - ServerName x.irspy.indexdata.com - ErrorLog /var/log/apache/irspy-error.log - CustomLog /var/log/apache/irspy-access.log combined - DocumentRoot /usr/local/src/cvs/irspy/web/htdocs - - SetHandler perl-script - PerlHandler HTML::Mason::ApacheHandler - - PerlAddVar MasonCompRoot "private => /usr/local/src/cvs/irspy/web/htdocs" - PerlSetVar MasonDataDir /usr/local/src/cvs/irspy/web/data - PerlSetVar IRSpyLibDir /usr/local/src/cvs/irspy/lib - PerlSetEnv PERL5LIB /usr/local/src/cvs/irspy/lib - - AuthType Basic - AuthName "IRSpy Administration" - AuthUserFile /usr/local/src/cvs/irspy/web/conf/htpasswd - Require user admin - - diff --git a/web/conf/apache2.0/cfspy.conf b/web/conf/apache2.0/cfspy.conf deleted file mode 100644 index 10d4622..0000000 --- a/web/conf/apache2.0/cfspy.conf +++ /dev/null @@ -1,52 +0,0 @@ -# $Id: cfspy.conf,v 1.4 2009-04-16 18:31:20 wosch Exp $ -# -# Sample configuration for running an IRSpy web-site under Apache 2.x. -# -# This is the configuration that I use on the development machine us2 -# /home/wosch/indexdata/irspy -# -# This file can be included in the Apache configuration by adding a -# line like this to apache.conf or one of its included files: -# Include /home/wosch/indexdata/irspy/web/conf/apache2.0/cfspy.conf -# -# When running on Ubuntu (and probably Debian), don't forget to -# install libapache-request-perl as well as libhtml-mason-perl, -# otherwise Mason will fail mysteriously. -# -# For Apache 2.x -# sudo apt-get install libxml-libxslt-perl -# sudo apt-get install libhtml-mason-perl -# sudo apt-get install libapache2-request-perl - -#NameVirtualHost 192.168.1.75 - - - ServerName cfspy.indexdata.com - - ErrorLog /var/log/apache2/cfspy-error.log - CustomLog /var/log/apache2/cfspy-access.log combined - DocumentRoot /home/wosch/indexdata/irspy/web/htdocs - - SetHandler perl-script - PerlHandler HTML::Mason::ApacheHandler - - - PerlSetVar MasonArgsMethod mod_perl - - PerlAddVar MasonCompRoot "private => /home/wosch/indexdata/irspy/web/htdocs" - PerlSetVar MasonDataDir /home/wosch/indexdata/irspy/web/data - PerlSetVar IRSpyLibDir /home/wosch/indexdata/irspy/lib - PerlSetEnv PERL5LIB /home/wosch/indexdata/irspy/lib - - # We need +Parent to make PerlSwitches -I work properly ... don't ask. - PerlOptions +Parent - PerlSwitches -I/home/wosch/indexdata/irspy/lib - - - AuthType Basic - AuthName "IRSpy Administration" - AuthUserFile /home/wosch/indexdata/irspy/web/conf/htpasswd - Require user admin - - - diff --git a/web/conf/apache2.0/irspy.conf b/web/conf/apache2.0/irspy.conf new file mode 100644 index 0000000..d7bc973 --- /dev/null +++ b/web/conf/apache2.0/irspy.conf @@ -0,0 +1,45 @@ +# $Id: cfspy.conf,v 1.4 2009-04-16 18:31:20 wosch Exp $ +# +# Sample configuration for running an IRSpy web-site under Apache 2.x. +# +# This is the configuration that I use on the development machine +# /usr/local/src/git/irspy +# +# This file can be included in the Apache configuration by adding a +# line like this to apache.conf or one of its included files: +# Include /usr/local/src/git/irspy/web/conf/apache2.0/irspy.conf +# +# For debian & unbuntu: please read irspy/README for a list of depending packages +# + + + ServerName irspy.indexdata.com + ServerAlias irspy2.indexdata.com irspy + + ErrorLog /var/log/apache2/irspy-error.log + CustomLog /var/log/apache2/irspy-access.log combined + DocumentRoot /usr/local/src/git/irspy/web/htdocs + + SetHandler perl-script + PerlHandler HTML::Mason::ApacheHandler + + + PerlSetVar MasonArgsMethod mod_perl + + PerlAddVar MasonCompRoot "private => /usr/local/src/git/irspy/web/htdocs" + PerlSetVar MasonDataDir /usr/local/src/git/irspy/web/data + PerlSetVar IRSpyLibDir /usr/local/src/git/irspy/lib + PerlSetEnv PERL5LIB /usr/local/src/git/irspy/lib + + # We need +Parent to make PerlSwitches -I work properly ... don't ask. + PerlOptions +Parent + PerlSwitches -I/usr/local/src/git/irspy/lib + + + AuthType Basic + AuthName "IRSpy Administration" + AuthUserFile /usr/local/src/git/irspy/web/conf/htpasswd + Require user admin + + + diff --git a/web/data/cache/.cvsignore b/web/data/cache/.cvsignore deleted file mode 100644 index b921e2e..0000000 --- a/web/data/cache/.cvsignore +++ /dev/null @@ -1 +0,0 @@ -private+2fdetails+2fstats.mc diff --git a/web/data/cache/.gitignore b/web/data/cache/.gitignore new file mode 100644 index 0000000..b921e2e --- /dev/null +++ b/web/data/cache/.gitignore @@ -0,0 +1 @@ +private+2fdetails+2fstats.mc diff --git a/web/data/obj/.cvsignore b/web/data/obj/.cvsignore deleted file mode 100644 index 873de48..0000000 --- a/web/data/obj/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -* -.__obj_create_marker diff --git a/web/data/obj/.gitignore b/web/data/obj/.gitignore new file mode 100644 index 0000000..873de48 --- /dev/null +++ b/web/data/obj/.gitignore @@ -0,0 +1,2 @@ +* +.__obj_create_marker diff --git a/web/logs/.cvsignore b/web/logs/.cvsignore deleted file mode 100644 index cbe462c..0000000 --- a/web/logs/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -access.log -error.log diff --git a/web/logs/.gitignore b/web/logs/.gitignore new file mode 100644 index 0000000..cbe462c --- /dev/null +++ b/web/logs/.gitignore @@ -0,0 +1,2 @@ +access.log +error.log diff --git a/xsl/irspy2zeerex.xsl b/xsl/irspy2zeerex.xsl index 0076589..76ad5a1 100644 --- a/xsl/irspy2zeerex.xsl +++ b/xsl/irspy2zeerex.xsl @@ -183,6 +183,7 @@ + diff --git a/zebra/Makefile b/zebra/Makefile index 481a125..0edf0da 100644 --- a/zebra/Makefile +++ b/zebra/Makefile @@ -1,4 +1,18 @@ -# $Id: Makefile,v 1.4 2007-09-13 12:19:58 mike Exp $ + +IRSPY_ARCHIVE= records-2010-04-06 +IRSPY_ARCHIVE_EXT= .tar.gz +DUMP_DIR= irspy-dump +ZEBRA_DIR= db +ZEBRAIDX= zebraidx-2.0 +ZEBRAIDX_TEST= zebraidx-2.0 -c zebra-test.cfg + +# debugging +ZEBRA_TEST_DIR= db-test + +DATE:= $(shell date '+%Y-%m-%d') + +all: help + test: xmllint --noout --schema zeerex-2.0.xsd zeerex.xml @@ -7,12 +21,35 @@ terse.properties: pqf.properties sed -n 's/#.*//; s/[ ]*//; s/[ ][ ]*/ /g; /./p' $? > $@ newdb: - tar xzf records-2007-05-01.tar.gz - zebraidx-2.0 init - zebraidx-2.0 update zeerex.xml - zebraidx-2.0 update records-2007-05-01 - zebraidx-2.0 commit + tar xzf ${DUMP_DIR}/${IRSPY_ARCHIVE}${IRSPY_ARCHIVE_EXT} + mkdir -p ${ZEBRA_DIR}/lock ${ZEBRA_DIR}/register ${ZEBRA_DIR}/shadow ${ZEBRA_DIR}/tmp + ${ZEBRAIDX} init + ${ZEBRAIDX} update zeerex.xml + ${ZEBRAIDX} update ${IRSPY_ARCHIVE} + ${ZEBRAIDX} commit + +newdb-test: + tar xzf ${DUMP_DIR}/${IRSPY_ARCHIVE}${IRSPY_ARCHIVE_EXT} + mkdir -p ${ZEBRA_TEST_DIR}/lock ${ZEBRA_TEST_DIR}/register ${ZEBRA_TEST_DIR}/shadow ${ZEBRA_TEST_DIR}/tmp + ${ZEBRAIDX_TEST} init + ${ZEBRAIDX_TEST} update zeerex-test.xml + ${ZEBRAIDX_TEST} update ${IRSPY_ARCHIVE} + ${ZEBRAIDX_TEST} commit +dump: + rm -rf records-${DATE}.old + -test -e records-${DATE} && mv records-${DATE} records-${DATE}.old + mkdir records-${DATE} + cd records-${DATE} && ../../bin/irspy-dump.pl localhost:8018/IR-Explain---1 + cd records-${DATE} && ( for i in *.xml; do tidy -xml -i -m -w 140 $$i; done ) > /dev/null 2>&1 + tar cf - records-${DATE} | gzip > records-${DATE}.tar.gz + clean: rm -f terse.properties +distclean: clean + rm -rf ${IRSPY_ARCHIVE} ${ZEBRA_DIR} ${ZEBRA_DIR} + +help: + @echo "make [ dump | test | newdb | newdb-test | clean | distclean ]" + diff --git a/zebra/README b/zebra/README index ad36e4d..2898e93 100644 --- a/zebra/README +++ b/zebra/README @@ -100,9 +100,14 @@ The database can be interrogated with SRU URLs such as: To create the database: +$ make newdb + +or: + tar xzf records-2007-04-18.tar.gz zebraidx-2.0 init zebraidx-2.0 update zeerex.xml -#zebraidx-2.0 update records-2010-04-06 +zebraidx-2.0 update record-2010-04-06 zebraidx-2.0 commit + zebrasrv-2.0 -f yazserver.xml diff --git a/zebra/init-script b/zebra/init-script index 4f9b223..0735ae0 100755 --- a/zebra/init-script +++ b/zebra/init-script @@ -1,6 +1,5 @@ #! /bin/sh -# $Id: init-script,v 1.8 2009-04-15 18:16:46 wosch Exp $ # # This is a startup/shutdown script for IRSpy's Zebra server, suitable # for use in a SysV-style init directory such as /etc/init.d (on @@ -19,10 +18,10 @@ logfile=/var/log/irspy-zebra pidfile=/var/run/irspy-zebra.pid case "`hostname`" in - xeno|rafa|test) IRSPYDIR=/usr/local/src/cvs/irspy/;; + nigiri | xeno | rafa | test) IRSPYDIR=/usr/local/src/git/irspy/;; shawarma|kafta) IRSPYDIR=/home/mike/cvs/irspy/;; - us2| us2.*) - IRSPYDIR=/home/wosch/indexdata/irspy/ + your-development-machine ) + IRSPYDIR=/home/foobar/indexdata/irspy/ logfile=$IRSPYDIR/irspy-zebra pidfile=$IRSPYDIR/irspy-zebra.pid ;; @@ -60,3 +59,4 @@ case "$1" in echo "Usage: $0 start|stop|restart" >&2 ;; esac + diff --git a/zebra/irspy-dump/records-2007-04-18.tar.gz b/zebra/irspy-dump/records-2007-04-18.tar.gz new file mode 100644 index 0000000..61a193d Binary files /dev/null and b/zebra/irspy-dump/records-2007-04-18.tar.gz differ diff --git a/zebra/irspy-dump/records-2007-05-01.tar.gz b/zebra/irspy-dump/records-2007-05-01.tar.gz new file mode 100644 index 0000000..631f551 Binary files /dev/null and b/zebra/irspy-dump/records-2007-05-01.tar.gz differ diff --git a/zebra/irspy-dump/records-2008-09-16.tar.gz b/zebra/irspy-dump/records-2008-09-16.tar.gz new file mode 100644 index 0000000..8d20960 Binary files /dev/null and b/zebra/irspy-dump/records-2008-09-16.tar.gz differ diff --git a/zebra/irspy-dump/records-2010-04-06.tar.gz b/zebra/irspy-dump/records-2010-04-06.tar.gz new file mode 100644 index 0000000..ef5319d Binary files /dev/null and b/zebra/irspy-dump/records-2010-04-06.tar.gz differ diff --git a/zebra/irspy-dump/records-2010-04-14.tar.gz b/zebra/irspy-dump/records-2010-04-14.tar.gz new file mode 100644 index 0000000..32dd0d8 Binary files /dev/null and b/zebra/irspy-dump/records-2010-04-14.tar.gz differ diff --git a/zebra/records-2007-04-18.tar.gz b/zebra/records-2007-04-18.tar.gz deleted file mode 100644 index 61a193d..0000000 Binary files a/zebra/records-2007-04-18.tar.gz and /dev/null differ diff --git a/zebra/records-2007-05-01.tar.gz b/zebra/records-2007-05-01.tar.gz deleted file mode 100644 index 631f551..0000000 Binary files a/zebra/records-2007-05-01.tar.gz and /dev/null differ diff --git a/zebra/records-2008-09-16.tar.gz b/zebra/records-2008-09-16.tar.gz deleted file mode 100644 index 8d20960..0000000 Binary files a/zebra/records-2008-09-16.tar.gz and /dev/null differ diff --git a/zebra/yazserver-test.xml b/zebra/yazserver-test.xml new file mode 100644 index 0000000..a642390 --- /dev/null +++ b/zebra/yazserver-test.xml @@ -0,0 +1,21 @@ + + + + tcp:@:8019 + + ./ + zebra-test.cfg + pqf.properties + + + + + localhost + 8019 + IR-Explain---2 + + + + + + diff --git a/zebra/zebra-test.cfg b/zebra/zebra-test.cfg new file mode 100644 index 0000000..6169c28 --- /dev/null +++ b/zebra/zebra-test.cfg @@ -0,0 +1,32 @@ + +# Where to look for config files +#profilePath: .:/usr/local/share/idzebra-2.0/tab:/usr/share/idzebra-2.0/tab +profilePath: .:/usr/share/idzebra-2.0/tab + +# Where to look for loadable zebra modules. Both these path components +# are necessary, since the former is used when installing from a Debian +# package and the latter when installing from a CVS build. +modulePath: /usr/lib/idzebra-2.0/modules +#modulePath: /usr/local/lib/idzebra-2.0/modules + +# store records and record keys internally +storeData: 1 +storeKeys: 1 + +# Use the "alvis" filter with config file "filterconf.xml" +recordtype: alvis.filterconf.xml + +database: IR-Explain---2 +#database: Default + +# where to put registers, and other var content, and how large they may be +register: db-test/register:100G +shadow: db-test/shadow:100G +lockdir: db-test/lock +keytmpdir: db-test/tmp + +# Permissions for update +perm.anonymous: ra +perm.admin: rw +passwd.c: htpasswd + diff --git a/zebra/zebra.cfg b/zebra/zebra.cfg index 2e8a28a..4e161a1 100644 --- a/zebra/zebra.cfg +++ b/zebra/zebra.cfg @@ -1,7 +1,7 @@ -# $Id: zebra.cfg,v 1.12 2006-12-19 12:53:16 sondberg Exp $ # Where to look for config files -profilePath: .:/usr/local/share/idzebra-2.0/tab:/usr/share/idzebra-2.0/tab +#profilePath: .:/usr/local/share/idzebra-2.0/tab:/usr/share/idzebra-2.0/tab +profilePath: .:/usr/share/idzebra-2.0/tab # Where to look for loadable zebra modules. Both these path components # are necessary, since the former is used when installing from a Debian @@ -29,3 +29,4 @@ keytmpdir: db/tmp perm.anonymous: ra perm.admin: rw passwd.c: htpasswd + diff --git a/zebra/zeerex-test.xml b/zebra/zeerex-test.xml new file mode 100644 index 0000000..95a25c1 --- /dev/null +++ b/zebra/zeerex-test.xml @@ -0,0 +1,172 @@ + + + + + + localhost + 8019 + IR-Explain---2 + + + + IRSpy database of ZeeRex records + + This is a database of enhanced ZeeRex records, maintained by IRSpy, + containing information about a selection of Z39.50 and SRU/SRW + databases that are periodically probed. The ZeeRex records are + extended using the extension mechanism described at + http://explain.z3950.org/dtd/extensions.html + so that the results of the tests are included alongside the + protocol-level information that is the core of ZeeRex. + + Index Data Aps, http://www.indexdata.com/ + Mike Taylor, mike@indexdata.com + + The initial version of this database was populated using the set of + targets known in Index Data's Z39.50 Target Directory: + http://targettest.indexdata.com/ + The records may have been substantially altered and extended since + then, and new records may have been added. + + + + Index Data's Zebra information management system: + http://www.indexdata.dk/zebra + + + + + + + CQL + + + Record Metadata + + + Network Metadata + + + Dublin Core + + + ZeeRex + + + BIB-1 + + + + Server choice + serverChoice + anywhere + cql:all + + + + Record identifier + id + rec:id + 3 + + + + Protocol + protocol + net:protocol + + + + Protocol version + version + net:version + 3 + + + + Protocol method + method + net:method + + + + Host + host + net:host + 3 + + + + Port + port + net:port + 3 + + + + Path + path + net:path + + + + Title + title + dc:title + + + + Creator + creator + dc:creator + + + + + + + + ZeeRex 2.0 + + + Zebra indexing recipe + + + Record identifier + + + + + = + relevant + fuzzy + regexp + * + ? + net + cql.anywhere + + + + + + 81 + Zebra Information Server/GFS/YAZ + Zebra 2.0.1/1.98/2.1.30 + + 2006-07-10T11:45:06 + 2006-07-17T11:45:02 + + + +