# Depends heavily on having ssh key authentication set up to all
# boxes. That's why I run it on my own workstation.
#
+# Regular debian upgrades are detected by running
+# apt-get upgrade -s
+# on every machine, and parsing the output.
+
+# We have decided to maintain some packages manually on some
+# machines, so that system-level upgrades will not disturb
+# applications, which may need more hand-holding. These are
+# extracted from our apt repository, and queried on every
+# server with apt-cache policy. This way, as soon as a package
+# is released on our repo, it will get listed here.
+#
# 11-Mar-2011 Heikki: Started this
+# 22-Mar-2011 Heikki: Adding manually maintained packages
+# 15-Aug-2011 Heikki: Adding a total in the headline, for nagiosgrapher
+#
+# TODO: Assumes that we release our restricted packages for all versions
+# and architectures at the same time. Gets only the highest version from
+# all, and reports anything less than this. Good enough for now.
+#
+# TODO: Get the dates from ls --full-time /var/cache/apt/archives/
+# and display next to the packages, so we can see how long they have
+# been lingering. Boldface them if older than some limit
#### Init
+use strict;
my $debug= $ARGV[0] || 0; # 0=none, 1=some, 2=more, 3=much
my $year =`date +%Y`;
my $wikilink = 'http://twiki.indexdata.dk/cgi-bin/twiki/view/ID/';
+my $restrictedpackages = "ssh -q kebab cat /home/ftp/pub/debian/dists/*/restricted/*/Packages";
+
+#### Host comments
+my %hostcomments = (
+ "ariel" => "<i>Niels Erik</i> does the manual upgrades",
+ "bellone" => "<i>Niels Erik</i> does the manual upgrades",
+ "cfrepous" => "<i>Wolfram</i> does the manual upgrades",
+ "leopard" => "<i>Wolfram</i> does the manual upgrades",
+ "lsd" => "<i>Heikki</i> takes care of all upgrades",
+ );
+
#### Get list of hosts
# I could use a hard-coded list, but I would forget to maintain it.
-# Nagios knows most of our hosts.
+# Nagios knows most of our hosts. It even knows which are worth
+# checking, they have a command to check apts!
my $hostlist = `ssh nagios grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg`
or die "Could not get host list";
print "Got list:\n$hostlist\n" if $debug>2;
+###### Get list of packages that can be manually maintained
+print "getting restricted package versions\n" if $debug;
+my %restrpkgs;
+my $restplines = `$restrictedpackages`
+ or die "Could not get the list of restricted packages " .
+ "from $restrictedpackages: $! ";
+print "Got package list: \n$restplines\n" if $debug>2;
+my $pname;
+my $pver;
+for my $pline ( split("\n",$restplines) ) {
+ chomp($pline);
+ $pname = $1 if $pline =~ /^Package:\s+(\S*)\s*$/;
+ $pver = $1 if $pline =~ /^Version:\s+(\S*)\s*$/;
+ print "$pline: p=$pname v=$pver\n" if $debug>2;
+ if ( $pname && $pver ) {
+ print "\nPackage $pname version $pver \n" if $debug>2;
+ if ( ! $restrpkgs{$pname} ) {
+ $restrpkgs{$pname} = $pver;
+ print "found $pname, first version $pver\n" if $debug>1;
+ } else {
+ my $bver = $restrpkgs{$pname};
+ `dpkg --compare-versions $bver lt $pver`;
+ if ( ! $? ) {
+ print "found $pname, better version $pver (better than $bver)\n"
+ if $debug>1;
+ $restrpkgs{$pname} = $pver;
+ } else {
+ print "found $pname, but version $pver is no better than $bver\n"
+ if $debug>2;
+ }
+ }
+ $pname = ""; # clear for the next one.
+ $pver = "";
+ }
+}
+
+if ( $debug >1 ) {
+ print "got " . scalar(keys(%restrpkgs)) . " restricted packages\n";
+ for $pname ( sort (keys(%restrpkgs)) ) {
+ print " $pname " . $restrpkgs{$pname} . "\n";
+ }
+}
+
# Statistics
my %summary;
-my %sechosts;
-my %secpkgs;
-my %ownhosts;
-my %ownpkgs;
-my %normhosts;
-my %normpkgs;
+my ( %sechosts, %secpkgs );
+my ( %ownhosts, %ownpkgs );
+my ( %manhosts, %manpkgs );
+my ( %normhosts, %normpkgs );
my %okhosts;
my %skiphosts;
+my %allhosts;
my $sectot = 0;
my $owntot = 0;
+my $mantot = 0;
my $normtot = 0;
my $table = "<table>\n";
-for $hline ( split("\n",$hostlist) ) {
+for my $hline ( split("\n",$hostlist) ) {
next unless ( $hline =~ /\/([a-z0-9-]+)\.cfg$/ );
my $H = $1;
next if ($H =~ /^commands/ );
next if ($H =~ /^servicegroups/ );
print "Checking $H\n" if $debug;
- my $apt = `ssh $H apt-get upgrade -s -o 'Debug::NoLocking=true' `;
+ $allhosts{$H}=1;
+ my $cmd1 = "apt-cache -q policy " . join(" ",sort(keys(%restrpkgs)));
+ my $cmd2 = "apt-get upgrade -s -o 'Debug::NoLocking=true' ";
# Note, do not append -qq, we want some output even when nothing to do
+ my $apt = `ssh -q $H "$cmd1 ; $cmd2 " 2>/dev/null`;
if ( !$apt ) {
$table .= "<tr><td colspan='3'> </td></tr>\n";
$table .= "<tr><td colspan='3'><b><u>$H</u></b> (skipped)\n";
next;
}
print "Got apts for $H: \n$apt\n" if $debug>2;
- my $det = "";
+ my $det = ""; # detail lines
my $pkgs = 0;
my $secs = 0;
my $own = 0;
- for $p ( split("\n",$apt) ) {
+ my $man = 0;
+ my $restrname = "";
+ my $restrinst = "";
+ my $restrcand = "";
+ for my $p ( split("\n",$apt) ) {
+ # parse apt-cache output
+ $restrname = $1 if $p =~ /^(\S+):$/;
+ $restrinst = $1 if $p =~ /^\s+Installed:\s+(\S+)$/;
+ $restrcand = $1 if $p =~ /^\s+Candidate:\s+(\S+)$/;
+ if ( $p =~ /^\s+Version table:/ ) { # have all for that package
+ my $bver = $restrpkgs{$restrname};
+ if ( ( $restrinst eq $restrcand ) &&
+ ( $restrinst ne $bver ) ) {
+ # if different, it is a regular apt upgrade, and will be seen
+ # later. AND we want to have a different version in our repo
+ `dpkg --compare-versions $bver lt $restrinst`;
+ if ( $? ) { # It was not a downgrade
+ # manual packages may be ahead of the repo!
+ $mantot++;
+ $man++;
+ $pkgs++;
+ $manhosts{$H} = 1;
+ $manpkgs{$restrname} = 1;
+ $det .= "<tr>";
+ $det .= "<td> $restrname <b>(M)</b></td>";
+ $det .= "<td>". strdiff($bver,$restrinst)."</td>";
+ $det .= "<td>". strdiff($restrinst,$bver)."</td>";
+ $det .= "</tr>\n";
+ my $key = "$restrname";
+ if ( !$summary{$key} ) {
+ $summary{$key} = "";
+ }
+ $summary{$key} .= "$H ";
+ }
+ }
+ $restrname = ""; # clear for next round
+ $restrinst = "";
+ $restrcand = "";
+ }
next unless $p =~
/^Inst ([^ ]+) \[([^]]+)\] \(([^ ]+) ([^:]+):/;
my ( $pkg,$cur,$new,$src ) = ( $1,$2,$3,$4 );
$pkgs++;
my $key = $pkg;
if ( $src =~ /Security/ ) {
- $det .= "<b>$pkg</b>";
- $key = "<b>$pkg</b>";
- $sechosts{$H}=1;
- $secpkgs{$pkg}=1;
+ $det .= "<b>$pkg (s)</b> ";
+ $sechosts{$H} = 1;
+ $secpkgs{$pkg} = 1;
$secs++;
$sectot++;
} elsif ( $src =~ /Indexdata/ ) {
- $det .= "<b><i>$pkg</i></b>";
- $key = "<i>$pkg</i>";
+ $det .= "<i><b>$pkg</b> (id) </i>";
$ownhosts{$H}=1;
$ownpkgs{$pkg}=1;
$own++;
$owntot++;
} else {
- $det .= "$pkg";
+ $det .= "$pkg ";
$normhosts{$H}=1;
$normpkgs{$pkg}=1;
$normtot++;
if ( !$summary{$key} ) {
$summary{$key} = "";
}
+ $summary{$key} .= "$H ";
$new = strdiff($cur,$new);
$cur = strdiff($new,$cur);
- $summary{$key} .= "$H ";
$det .= "</td> ";
$det .= "<td>$cur</td> ";
$det .= "<td>$new</td> ";
if ( $pkgs ) {
$table .= "<b>$pkgs</b> packages to upgrade. ";
$table .= "<b>$secs security</b>. " if $secs;
- $table .= " $own from indexdata " if $own;
+ $table .= " $own from indexdata. " if $own;
+ $table .= " $man manual. " if $man;
} else {
$table .= "ok";
+ $okhosts{$H} = 1;
}
my $updlink = $wikilink . ucfirst($H) . "Updates" . $year;
+ # Fix some pages that do not follow the convention.
+ # Mostly because the host names would not make proper WikiWords
+ $updlink =~ s/Bugzilla3Updates/BugzillaUpdates/;
+ $updlink =~ s/Opencontent-solrUpdates/OpenContentSolrUpdates/;
$table .= " <a href='$updlink' >Upd</a>";
$table .= "</td></tr>\n";
+ $table .= "<tr><td>$hostcomments{$H}</td></tr>\n"
+ if ( $hostcomments{$H} );
$table .= $det if $pkgs;
print "\n$table\n" if $debug>2;
- last if $H =~/dart/ && $debug;
+ last if $H =~/diane/ && $debug;
}
$table .= "</table>\n";
-# Produce page
+# Page header
my $outfile = "/tmp/aptcheck.html";
open F, ">$outfile"
or die "Could not open $outfile for writing: $!";
print F "<head><title>Apt upgrade status</title></head>\n";
print F "<body>\n";
print F "<H1>Apt package status</H1>\n";
+print F "<b>" . ( $sectot + $owntot + $mantot + $normtot ) .
+ "</b> packages pending (<b>$sectot</b> critical) \n";
+print F "<H2>Debug run, many hosts missing!</H2>\n"
+ if $debug;
-# Summary table
+
+# Summary table: one row for per host group
+print F "<p/>\n";
print F "<table border='1' >\n";
print F "<tr><td> </td>" ;
-print F "<td><b>Security</b><br/>" . scalar(keys(%sechosts)) .
- " / " . scalar(keys(%secpkgs)) . " / $sectot </td>\n" ;
-print F "<td>Indexdata<br/>" . scalar(keys(%ownhosts)) .
- " / " . scalar(keys(%ownpkgs)) . " / $owntot </td>\n" ;
-print F "<td>Normal<br/>" . scalar(keys(%normhosts)) .
- " / " . scalar(keys(%normpkgs)) . " / $normtot </td>" . "</tr>\n";
-print F "<tr><td>Hosts</td>\n";
-
-
-print F "<td>";
-for $HH ( sort(keys(%sechosts)) ) {
- print F "<a href='#$HH'><b>$HH</b></a> ";
+print F "<td><b>Hosts</b></td>\n";
+print F "<td><b>Packages</b></td></tr>\n";
+
+if ( $sectot ) {
+ print F "<tr><td><b>Security</b><br/>" . scalar(keys(%sechosts)) .
+ " / " . scalar(keys(%secpkgs)) . " / $sectot </td>\n" ;
+ print F "<td>";
+ for my $HH ( sort(keys(%sechosts)) ) {
+ print F "<a href='#$HH'><b>$HH</b></a> ";
+ }
+ print F "</td>";
+ print F "<td>";
+ for my $PP ( sort(keys(%secpkgs)) ) {
+ print F "<a href='#$PP'>$PP</a> ";
+ }
+ print F "</td>";
+ print F "</tr>\n";
+}
+if ( $owntot ) {
+ print F "<tr><td><b>Indexdata</b><br/>" . scalar(keys(%ownhosts)) .
+ " / " . scalar(keys(%ownpkgs)) . " / $owntot </td>\n" ;
+ print F "<td>";
+ for my $HH ( sort(keys(%ownhosts)) ) {
+ print F "<a href='#$HH'><b>$HH</b></a> ";
+ }
+ print F "</td>";
+ print F "<td>";
+ for my $PP ( sort(keys(%ownpkgs)) ) {
+ print F "<a href='#$PP'>$PP</a> ";
+ }
+ print F "</td>";
+ print F "</tr>\n";
}
-print F " </td>\n";
-#print F "<td><b>" . join(" ",sort(keys(%sechosts))) . "</b> </td>";
-#print F "<td><b>" . join(" ",sort(keys(%ownhosts))) . "</b> </td>";
-print F "<td>";
-for $HH ( sort(keys(%ownhosts)) ) {
- print F "<a href='#$HH'><b>$HH</b></a> ";
+if ( $mantot ) {
+ print F "<tr><td><b>Manual</b><br/>" . scalar(keys(%manhosts)) .
+ " / " . scalar(keys(%manpkgs)) . " / $mantot </td>\n" ;
+ print F "<td>";
+ for my $HH ( sort(keys(%manhosts)) ) {
+ print F "<a href='#$HH'><b>$HH</b></a> ";
+ }
+ print F "</td>";
+ print F "<td>";
+ for my $PP ( sort(keys(%manpkgs)) ) {
+ print F "<a href='#$PP'>$PP</a> ";
+ }
+ print F "</td>";
+ print F "</tr>\n";
}
-print F " </td>\n";
-#print F "<td><b>" . join(" ",sort(keys(%normhosts))) . "</b> </td></tr>\n";
-print F "<td>";
-for $HH ( sort(keys(%normhosts)) ) {
- print F "<a href='#$HH'><b>$HH</b></a> ";
+if ( $normtot ) {
+ print F "<tr><td>Normal<br/>" . scalar(keys(%normhosts)) .
+ " / " . scalar(keys(%normpkgs)) . " / $normtot </td>\n" ;
+ print F "<td>";
+ for my $HH ( sort(keys(%normhosts)) ) {
+ print F "<a href='#$HH'><b>$HH</b></a> ";
+ }
+ print F "</td>";
+ print F "<td>";
+ for my $PP ( sort(keys(%normpkgs)) ) {
+ print F "<a href='#$PP'>$PP</a> ";
+ }
+ print F "</td>";
+ print F "</tr>\n";
+}
+if ( %skiphosts ) {
+ print F "<tr><td>Skipped " . scalar(keys(%skiphosts)) . "</td>\n";
+ print F "<td colspan='2'>";
+ for my $HH ( sort(keys(%skiphosts)) ) {
+ print F "<a href='#$HH'><b>$HH</b></a> ";
+ }
+ print F "</td></tr>\n";
+}
+#if ( %okhosts ) {
+if ( 1 ) {
+ print F "<tr><td>Ok " . scalar(keys(%okhosts)) . "</td>\n";
+ print F "<td colspan='2'>";
+ for my $HH ( sort(keys(%okhosts)) ) {
+ print F "<a href='#$HH'><b>$HH</b></a> ";
+ }
+ if ( !%okhosts ) {
+ print F "<b>None at all!</b>";
+ }
+ print F "</td></tr>\n";
}
-print F " </td>\n";
-
-print F "<tr><td>Packages</td>\n";
-print F "<td>" . join(" ",sort(keys(%secpkgs))) . " </td>";
-print F "<td>" . join(" ",sort(keys(%ownpkgs))) . " </td>";
-print F "<td>" . join(" ",sort(keys(%normpkgs))) . " </td></tr>\n";
print F "</table>\n";
+# Graph
+#my $secs = 60*60*24 * 7 * 2; # 2 weeks in secods
+#my $secs = "1m"; # one month, let nagios do the math
+my $secs = "45d";
+print F "<p/>" .
+ "<a href='http://nagios.indexdata.com/cgi-bin/nagios3/graphs.cgi?" .
+ "host=nagios&service=Apt%20Summary'>\n".
+ "<img src='http://nagios.indexdata.com/" .
+ "cgi-bin/nagios3/rrd2-system.cgi?" .
+ "host=nagios&service=Apt%20Summary&" .
+ "start=-$secs&" .
+ "width=800&height=100&type=AVERAGE' /> ".
+ "</a>" .
+ "<br/>\n";
+
+# The host table
print F $table;
+# Package table
+print F "<p/><b><u>Packages</u></b>\n";
+print F "<table>\n";
+for my $P ( sort(keys(%summary)) ) {
+ my $PN = $P;
+ $PN = "<b>$P (s)</b>" if ($secpkgs{$P});
+ $PN = "<i>$P (id)</i>" if ($ownpkgs{$P});
+ $PN = "$P <b>(M)</b>" if ($manpkgs{$P});
+ print F "<tr><td><a name='$P'/>$PN</td>\n";
+ print F "<td>";
+ for my $HH ( split(' ',$summary{$P} )) {
+ print F "<a href=#$HH>$HH</a> ";
+ }
+ print F "</td>\n";
+
+}
+print F "</table>\n";
+
print F "<p/>Produced " . `date`.
" on " . `hostname` . " by " . `whoami` .
"<br/>\n";
system "scp -q $outfile nagios:/var/www/heikki/index.html";
+exit(0);
+
# Helper to take two strings and highligt that part of the second
# that is different from the first.
sub strdiff {
my $x = shift;
my $y = shift;
- print "strdiff: '$x' '$y' \n" if $debug>1;
+ print "strdiff: '$x' '$y' \n" if $debug>2;
+ if ( $x eq $y ) {
+ return "$x <b>??</b>";
+ }
my $a = 0;
while ( $a < length($y) &&
substr($x,$a,1) eq substr($y,$a,1) ) {
$a++;
}
if ( $a == length($y) ) {
- return "$y ???";
+ return "$y";
}
my $b = 1;
while ( $b < length($y)-$a &&
$b++;
}
my $c = length($y) - $b +1;
- print "strdiff: a=$a " . substr($y,0,$a) ."\n" if $debug>1;
- print "strdiff: b=$b " . "\n" if $debug>1;
- print "strdiff: c=$c " . substr($y,$c) ."\n" if $debug>1;
- print "strdiff: " . substr($y,$a, $c-$a) ."\n" if $debug>1;
+ print "strdiff: a=$a " . substr($y,0,$a) ."\n" if $debug>2;
+ print "strdiff: b=$b " . "\n" if $debug>2;
+ print "strdiff: c=$c " . substr($y,$c) ."\n" if $debug>2;
+ print "strdiff: " . substr($y,$a, $c-$a) ."\n" if $debug>2;
my $z = substr($y,0,$a) .
"<b>" . substr($y,$a, $c-$a) . "</b>" .
substr($y,$c);
- print "strdiff: " . $z ."\n" if $debug>1;
- print "\n" if $debug>1;
+ print "strdiff: " . $z ."\n" if $debug>2;
+ print "\n" if $debug>2;
return $z;
}