Merge branch 'master' of ssh://git.indexdata.com:222/home/git/pub/git-tools

diff --git a/aptcheck/README b/aptcheck/README
new file mode 100644 (file)
index 0000000..5fad11c
--- /dev/null
+++ b/aptcheck/README
@@ -0,0 +1,6 @@
+Heikki's apt-upgrade check.
+
+Too small project to have its own git, so it lives here.
+
+See comments in the .pl file
+

diff --git a/aptcheck/aptcheck.pl b/aptcheck/aptcheck.pl
new file mode 100755 (executable)
index 0000000..7c9245b
--- /dev/null
+++ b/aptcheck/aptcheck.pl
@@ -0,0 +1,398 @@
+#!/usr/bin/perl -w
+#
+# Check what packages are needed to get upgraded on all machines
+#
+# Depends heavily on having ssh key authentication set up to all
+# boxes. That's why I run it on my own workstation.
+#
+# Regular debian upgrades are detected by running 
+#  apt-get upgrade -s
+# on every machine, and parsing the output.
+
+# We have decided to maintain some packages manually on some 
+# machines, so that system-level upgrades will not disturb 
+# applications, which may need more hand-holding. These are
+# extracted from our apt repository, and queried on every 
+# server with apt-cache policy. This way, as soon as a package
+# is released on our repo, it will get listed here.
+#
+# 11-Mar-2011 Heikki: Started this
+# 22-Mar-2011 Heikki: Adding manually maintained packages
+#
+# TODO: Assumes that we release our restricted packages for all versions
+# and architectures at the same time. Gets only the highest version from
+# all, and reports anything less than this. Good enough for now.
+
+#### Init
+use strict;
+my $debug= $ARGV[0] || 0; # 0=none, 1=some, 2=more, 3=much
+my $year =`date +%Y`;
+my $wikilink = 'http://twiki.indexdata.dk/cgi-bin/twiki/view/ID/';
+my $restrictedpackages = "ssh -q kebab cat /home/ftp/pub/debian/dists/*/restricted/*/Packages";
+
+#### Host comments
+my %hostcomments = (
+      "ariel"    => "<i>Niels Erik</i> does the manual upgrades",
+      "bellone"  => "<i>Niels Erik</i> does the manual upgrades",
+      "cfrepous" => "<i>Wolfram</i> does the manual upgrades",
+      "leopard"  => "<i>Wolfram</i> does the manual upgrades",
+      );
+      
+
+#### Get list of hosts
+# I could use a hard-coded list, but I would forget to maintain it.
+# Nagios knows most of our hosts. It even knows which are worth 
+# checking, they have a command to check apts!
+
+my $hostlist = `ssh nagios grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg`
+  or die "Could not get host list";
+
+print "Got list:\n$hostlist\n" if $debug>2;
+
+###### Get list of packages that can be manually maintained
+print "getting restricted package versions\n" if $debug;
+my %restrpkgs;
+my $restplines = `$restrictedpackages`
+  or die "Could not get the list of restricted packages " .
+         "from $restrictedpackages: $! ";
+print "Got package list: \n$restplines\n" if $debug>2;
+my $pname;
+my $pver;
+for my $pline ( split("\n",$restplines) ) {
+    chomp($pline);
+    $pname = $1 if $pline =~ /^Package:\s+(\S*)\s*$/;
+    $pver = $1 if $pline =~ /^Version:\s+(\S*)\s*$/;
+    print "$pline: p=$pname v=$pver\n" if $debug>2;
+    if ( $pname && $pver ) {
+        print "\nPackage $pname version $pver \n" if $debug>2;
+        if ( ! $restrpkgs{$pname} ) {
+            $restrpkgs{$pname} = $pver;
+            print "found $pname, first version $pver\n" if $debug>1;
+        } else {
+            my $bver = $restrpkgs{$pname};
+            `dpkg --compare-versions $bver lt $pver`;
+            if ( ! $? ) {
+                print "found $pname, better version $pver (better than $bver)\n"
+                    if $debug>1;
+                $restrpkgs{$pname} = $pver;
+            } else {
+                print "found $pname, but version $pver is no better than $bver\n"
+                    if $debug>2;
+            }
+        }
+        $pname = ""; # clear for the next one.
+        $pver = "";
+    }
+}
+
+if ( $debug >1 ) {
+    print "got " . scalar(keys(%restrpkgs)) . " restricted packages\n";
+    for $pname ( sort (keys(%restrpkgs)) ) {
+        print "  $pname " . $restrpkgs{$pname} . "\n";
+    }
+}
+
+# Statistics
+my %summary;
+my ( %sechosts, %secpkgs );
+my ( %ownhosts, %ownpkgs );
+my ( %manhosts, %manpkgs );
+my ( %normhosts, %normpkgs );
+my %okhosts;
+my %skiphosts;
+my %allhosts;
+my $sectot = 0;
+my $owntot = 0;
+my $mantot = 0;
+my $normtot = 0;
+
+my $table = "<table>\n";
+
+for my $hline ( split("\n",$hostlist) ) {
+    next unless ( $hline =~ /\/([a-z0-9-]+)\.cfg$/ );
+    my $H = $1;
+    next if ($H =~ /^commands/ );
+    next if ($H =~ /^servicegroups/ );
+    print "Checking $H\n" if $debug;
+    $allhosts{$H}=1;
+    my $cmd1 = "apt-cache -q policy " . join(" ",sort(keys(%restrpkgs)));
+    my $cmd2 = "apt-get upgrade -s -o 'Debug::NoLocking=true' ";
+    # Note, do not append -qq, we want some output even when nothing to do
+    my $apt = `ssh -q $H "$cmd1 ; $cmd2 " 2>/dev/null`;
+    if ( !$apt ) {
+        $table .= "<tr><td colspan='3'>&nbsp;</td></tr>\n";
+        $table .= "<tr><td colspan='3'><b><u>$H</u></b> (skipped)\n";
+        $skiphosts{$H}=1;
+        next;
+    }
+    print "Got apts for $H: \n$apt\n" if $debug>2;
+    my $det = ""; # detail lines
+    my $pkgs = 0;
+    my $secs = 0;
+    my $own = 0;
+    my $man = 0;
+    my $restrname = "";
+    my $restrinst = "";
+    my $restrcand = "";
+    for my $p ( split("\n",$apt) ) {
+        # parse apt-cache output
+        $restrname = $1 if $p =~ /^(\S+):$/;
+        $restrinst = $1 if $p =~ /^\s+Installed:\s+(\S+)$/;
+        $restrcand = $1 if $p =~ /^\s+Candidate:\s+(\S+)$/;
+        if ( $p =~ /^\s+Version table:/ ) { # have all for that package
+            my $bver = $restrpkgs{$restrname};
+            if ( ( $restrinst eq $restrcand ) &&
+                 ( $restrinst ne $bver ) ) { 
+                # if different, it is a regular apt upgrade, and will be seen
+                # later. AND we want to have a different version in our repo
+                `dpkg --compare-versions $bver lt $restrinst`;
+                if ( $? ) { # It was not a downgrade 
+                            # manual packages may be ahead of the repo!
+                    $mantot++;
+                    $man++;
+                    $pkgs++;
+                    $manhosts{$H} = 1;
+                    $manpkgs{$restrname} = 1;
+                    $det .= "<tr>";
+                    $det .= "<td>&nbsp;&nbsp;$restrname <b>(M)</b></td>";
+                    $det .= "<td>". strdiff($bver,$restrinst)."</td>";
+                    $det .= "<td>". strdiff($restrinst,$bver)."</td>";
+                    $det .= "</tr>\n";
+                    my $key = "$restrname";
+                    if ( !$summary{$key} ) {
+                        $summary{$key} = "";
+                    }
+                    $summary{$key} .= "$H ";
+                }
+            }
+            $restrname = ""; # clear for next round
+            $restrinst = "";
+            $restrcand = "";
+        }
+        next unless $p =~
+            /^Inst ([^ ]+) \[([^]]+)\] \(([^ ]+) ([^:]+):/;
+        my ( $pkg,$cur,$new,$src ) = ( $1,$2,$3,$4 );
+        print "$H: $pkg: $cur -> $new ($src)\n" if $debug>1;
+        $det .= "<tr><td>&nbsp;&nbsp;";
+        $pkgs++;
+        my $key = $pkg;
+        if ( $src =~ /Security/ ) {
+            $det .= "<b>$pkg (s)</b> ";
+            $sechosts{$H} = 1;
+            $secpkgs{$pkg} = 1;
+            $secs++;
+            $sectot++;
+        } elsif ( $src =~ /Indexdata/ ) {
+            $det .= "<i><b>$pkg</b> (id) </i>";
+            $ownhosts{$H}=1;
+            $ownpkgs{$pkg}=1;
+            $own++;
+            $owntot++;
+        } else {
+            $det .= "$pkg ";
+            $normhosts{$H}=1;
+            $normpkgs{$pkg}=1;
+            $normtot++;
+        }
+        if ( !$summary{$key} ) {
+            $summary{$key} = "";
+        }
+        $summary{$key} .= "$H ";
+        $new = strdiff($cur,$new);
+        $cur = strdiff($new,$cur);
+        $det .= "</td> ";
+        $det .= "<td>$cur</td> ";
+        $det .= "<td>$new</td> ";
+        $det .= "</tr>\n";
+    }
+    $table .= "<tr><td colspan='3'>&nbsp;</td></tr>\n";
+    $table .= "<tr><td colspan='3'><a name='$H'><b><u>$H</u></b></a> &nbsp;\n";
+    if ( $pkgs ) {
+        $table .= "<b>$pkgs</b> packages to upgrade. ";
+        $table .= "<b>$secs security</b>. " if $secs;
+        $table .= " $own from indexdata. " if $own;
+        $table .= " $man manual. " if $man;
+    } else {
+        $table .= "ok";
+        $okhosts{$H} = 1;
+    }
+    my $updlink = $wikilink . ucfirst($H) . "Updates" . $year;
+    # Fix some pages that do not follow the convention.
+    # Mostly because the host names would not make proper WikiWords
+    $updlink =~ s/Bugzilla3Updates/BugzillaUpdates/; 
+    $updlink =~ s/Opencontent-solrUpdates/OpenContentSolrUpdates/; 
+    $table .= "&nbsp;<a href='$updlink' >Upd</a>";
+    $table .= "</td></tr>\n";
+    $table .= "<tr><td>$hostcomments{$H}</td></tr>\n"
+        if ( $hostcomments{$H} );
+    $table .= $det if $pkgs;
+    print "\n$table\n" if $debug>2;
+    last if $H =~/dart/ && $debug;
+}
+$table .= "</table>\n";
+
+# Page header
+my $outfile = "/tmp/aptcheck.html";
+open F, ">$outfile"
+    or die "Could not open $outfile for writing: $!";
+print F "<html>\n";
+print F "<head><title>Apt upgrade status</title></head>\n";
+print F "<body>\n";
+print F "<H1>Apt package status</H1>\n";
+print F "<H2>Debug run, many hosts missing!</H2>\n"
+   if $debug;
+
+
+# Summary table: one row for per host group
+print F "<p/>\n";
+print F "<table border='1' >\n";
+print F "<tr><td>&nbsp;</td>" ;
+print F "<td><b>Hosts</b></td>\n";
+print F "<td><b>Packages</b></td></tr>\n";
+
+if ( $sectot ) {
+    print F "<tr><td><b>Security</b><br/>" . scalar(keys(%sechosts)) . 
+        "&nbsp;/&nbsp;" .  scalar(keys(%secpkgs)) . "&nbsp;/&nbsp;$sectot </td>\n" ;
+    print F "<td>";
+    for my $HH ( sort(keys(%sechosts)) ) {
+        print F "<a href='#$HH'><b>$HH</b></a> ";
+    }
+    print F "</td>";
+    print F "<td>";
+    for my $PP ( sort(keys(%secpkgs)) ) {
+        print F "<a href='#$PP'>$PP</a> ";
+    }
+    print F "</td>";
+    print F "</tr>\n";
+}
+if ( $owntot ) {
+    print F "<tr><td><b>Indexdata</b><br/>" . scalar(keys(%ownhosts)) . 
+        "&nbsp;/&nbsp;" .  scalar(keys(%ownpkgs)) . "&nbsp;/&nbsp;$owntot </td>\n" ;
+    print F "<td>";
+    for my $HH ( sort(keys(%ownhosts)) ) {
+        print F "<a href='#$HH'><b>$HH</b></a> ";
+    }
+    print F "</td>";
+    print F "<td>";
+    for my $PP ( sort(keys(%ownpkgs)) ) {
+        print F "<a href='#$PP'>$PP</a> ";
+    }
+    print F "</td>";
+    print F "</tr>\n";
+}
+if ( $mantot ) {
+    print F "<tr><td><b>Manual</b><br/>" . scalar(keys(%manhosts)) . 
+        "&nbsp;/&nbsp;" .  scalar(keys(%manpkgs)) . "&nbsp;/&nbsp;$mantot </td>\n" ;
+    print F "<td>";
+    for my $HH ( sort(keys(%manhosts)) ) {
+        print F "<a href='#$HH'><b>$HH</b></a> ";
+    }
+    print F "</td>";
+    print F "<td>";
+    for my $PP ( sort(keys(%manpkgs)) ) {
+        print F "<a href='#$PP'>$PP</a> ";
+    }
+    print F "</td>";
+    print F "</tr>\n";
+}
+if ( $normtot ) {
+    print F "<tr><td>Normal<br/>" . scalar(keys(%normhosts)) . 
+        "&nbsp;/&nbsp;" .  scalar(keys(%normpkgs)) . "&nbsp;/&nbsp;$normtot </td>\n" ;
+    print F "<td>";
+    for my $HH ( sort(keys(%normhosts)) ) {
+        print F "<a href='#$HH'><b>$HH</b></a> ";
+    }
+    print F "</td>";
+    print F "<td>";
+    for my $PP ( sort(keys(%normpkgs)) ) {
+        print F "<a href='#$PP'>$PP</a> ";
+    }
+    print F "</td>";
+    print F "</tr>\n";
+}
+if ( %skiphosts ) {
+    print F "<tr><td>Skipped " . scalar(keys(%skiphosts)) . "</td>\n";
+    print F "<td colspan='2'>";
+    for my $HH ( sort(keys(%skiphosts)) ) {
+        print F "<a href='#$HH'><b>$HH</b></a> ";
+    }
+    print F "</td></tr>\n";
+}
+if ( %okhosts ) {
+    print F "<tr><td>Ok " . scalar(keys(%okhosts)) . "</td>\n";
+    print F "<td colspan='2'>";
+    for my $HH ( sort(keys(%okhosts)) ) {
+        print F "<a href='#$HH'><b>$HH</b></a> ";
+    }
+    print F "</td></tr>\n";
+}
+print F "</table>\n";
+
+
+# The host table
+print F $table;
+
+# Package table
+print F "<p/><b><u>Packages</u></b>\n";
+print F "<table>\n";
+for my $P ( sort(keys(%summary)) ) {
+    my $PN = $P;
+    $PN = "<b>$P&nbsp;(s)</b>" if ($secpkgs{$P});
+    $PN = "<i>$P&nbsp;(id)</i>" if ($ownpkgs{$P});
+    $PN = "$P&nbsp;<b>(M)</b>" if ($manpkgs{$P});
+    print F "<tr><td><a name='$P'/>$PN</td>\n";
+    print F "<td>";
+    for my $HH ( split(' ',$summary{$P} )) {
+        print F "<a href=#$HH>$HH</a> ";
+    }
+    print F "</td>\n";
+
+}
+print F "</table>\n";
+
+print F "<p/>Produced " . `date`.
+        " on " . `hostname` . " by " . `whoami` .
+        "<br/>\n";
+print F "</body></html>\n";
+
+close(F)
+    or die "Could not close $outfile: $!";
+
+system "scp -q $outfile nagios:/var/www/heikki/index.html";
+
+exit(0);
+
+# Helper to take two strings and highligt that part of the second
+# that is different from the first. 
+sub strdiff {
+    my $x = shift;
+    my $y = shift;
+    print "strdiff: '$x' '$y' \n" if $debug>2;
+    if ( $x eq $y ) {
+        return "$x <b>??</b>";
+    }
+    my $a = 0;
+    while ( $a < length($y) &&
+        substr($x,$a,1) eq substr($y,$a,1) ) {
+        $a++;
+    }
+    if ( $a == length($y) ) {
+        return "$y";
+    }
+    my $b = 1;
+    while ( $b < length($y)-$a &&
+        substr($x,-$b,1) eq substr($y, -$b,1) ) {
+        $b++;
+    }
+    my $c = length($y) - $b +1;
+    print "strdiff:   a=$a " . substr($y,0,$a) ."\n" if $debug>2;
+    print "strdiff:   b=$b " . "\n" if $debug>2;
+    print "strdiff:   c=$c " . substr($y,$c) ."\n" if $debug>2;
+    print "strdiff:        " . substr($y,$a, $c-$a) ."\n" if $debug>2;
+    my $z =  substr($y,0,$a) .
+             "<b>" . substr($y,$a, $c-$a) . "</b>" .
+             substr($y,$c);
+    print "strdiff:        " . $z ."\n" if $debug>2;
+    print "\n" if $debug>2;
+    return $z;
+}

diff --git a/id-deb-build/id-upload.sh b/id-deb-build/id-upload.sh
index 57649b9..f6c08fe 100755 (executable)
--- a/id-deb-build/id-upload.sh
+++ b/id-deb-build/id-upload.sh
@@ -25,7 +25,7 @@ fi
 (cd doc && make ${PROD}.pdf index.html)
 cp NEWS doc/
 scp ${TAR} ftp.indexdata.dk:/home/ftp/pub/${PROD}/
-tar cz --exclude=.git -f - dox doc|ssh website "cd software/${PROD}; tar xzf -"
+tar cz --exclude=.git -f - dox doc|ssh website "cd /var/www/software/${PROD}; tar xzf -"
 # Local Variables:
 # mode:shell-script
 # sh-indentation: 2

diff --git a/id-deb-build/pbuilderrc b/id-deb-build/pbuilderrc
index 933fbe1..8d16d2b 100644 (file)
--- a/id-deb-build/pbuilderrc
+++ b/id-deb-build/pbuilderrc
@@ -7,7 +7,8 @@
 : ${ARCH:=$(dpkg --print-architecture)}
 NAME="$DIST-$ARCH"
 DISTRIBUTION="$DIST"
-DEBOOTSTRAPOPTS=("--arch" "$ARCH" "${DEBOOTSTRAPOPTS[@]}")
+# We avoid keyring in assignment below. bug 4122
+DEBOOTSTRAPOPTS=("--arch" "$ARCH" --variant=buildd)
 BASETGZ="`dirname $BASETGZ`/$NAME-base.tgz"
 #BUILDRESULT="/var/cache/pbuilder/$NAME/result/"
 BUILDRESULT="$NAME"

diff --git a/id-rpm-build/id-setup-rpmbuild.sh b/id-rpm-build/id-setup-rpmbuild.sh
new file mode 100755 (executable)
index 0000000..52376a2
--- /dev/null
+++ b/id-rpm-build/id-setup-rpmbuild.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+# Thie script will create an .rpmmacros file that will allow rpmbuild
+# as non-root. Results will be put in in rpmbuild of your home directory
+cd $HOME
+if test -f .rpmmacros; then
+       echo ".rpmmacros already exist"
+       exit 1
+fi
+echo "%_topdir $HOME/rpmbuild" >.rpmmacros
+mkdir -p rpmbuild/BUILD
+mkdir -p rpmbuild/RPMS/noarch
+mkdir -p rpmbuild/RPMS/x86_64
+mkdir -p rpmbuild/SOURCES
+mkdir -p rpmbuild/SPECS
+mkdir -p rpmbuild/SRPMS