X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;ds=sidebyside;f=aptcheck%2Faptcheck.pl;h=3121946ce18316788f8bf688984878e24602681e;hb=114cf0ae412ff4d23ff7488db783d1bd1ed8197b;hp=71e00fe5c003f3594c7e420f9de5ece4f733c112;hpb=2c2c89592cc515d7485732b89daa3c9e763c7c4c;p=git-tools-moved-to-github.git diff --git a/aptcheck/aptcheck.pl b/aptcheck/aptcheck.pl index 71e00fe..3121946 100755 --- a/aptcheck/aptcheck.pl +++ b/aptcheck/aptcheck.pl @@ -5,48 +5,167 @@ # Depends heavily on having ssh key authentication set up to all # boxes. That's why I run it on my own workstation. # +# Regular debian upgrades are detected by running +# apt-get upgrade -s +# on every machine, and parsing the output. + +# We have decided to maintain some packages manually on some +# machines, so that system-level upgrades will not disturb +# applications, which may need more hand-holding. These are +# extracted from our apt repository, and queried on every +# server with apt-cache policy. This way, as soon as a package +# is released on our repo, it will get listed here. +# # 11-Mar-2011 Heikki: Started this +# 22-Mar-2011 Heikki: Adding manually maintained packages +# 15-Aug-2011 Heikki: Adding a total in the headline, for nagiosgrapher +# 21-May-2012 Heikki: Added a date since when a package has been pending +# 31-May-2012 Heikki: Pointing to the new wiki +# 01-Jan-2013 Heikki: Get hosts from nagios-us as well. +# +# TODO: Assumes that we release our restricted packages for all versions +# and architectures at the same time. Gets only the highest version from +# all, and reports anything less than this. Good enough for now. +# #### Init +use strict; my $debug= $ARGV[0] || 0; # 0=none, 1=some, 2=more, 3=much my $year =`date +%Y`; -my $wikilink = 'http://twiki.indexdata.dk/cgi-bin/twiki/view/ID/'; +#my $wikilink = 'http://twiki.indexdata.dk/cgi-bin/twiki/view/ID/'; +my $wikilink = 'https://twiki.indexdata.com/twiki/bin/view/ID/'; +my $restrictedpackages = "ssh -q kebab cat /home/ftp/pub/debian/dists/*/restricted/*/Packages"; +my $updlink="-u"; # to display after a name, liking to the upd page + +#### Host comments +my %hostcomments = ( + "ariel" => "Niels Erik does the manual upgrades", + "bellone" => "Niels Erik does the manual upgrades", + "cfrepous" => "Wolfram does the manual upgrades", + "leopard" => "Wolfram does the manual upgrades", + "lsd" => "Heikki takes care of all upgrades", + ); + #### Get list of hosts # I could use a hard-coded list, but I would forget to maintain it. -# Nagios knows most of our hosts. +# Nagios knows most of our hosts. It even knows which are worth +# checking, they have a command to check apts! +print "Getting hostlist from nagios\n" if $debug; +my $hostlist1 = `ssh nagios grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg` + or die "Could not get host list from nagios (dk)"; -my $hostlist = `ssh nagios grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg` - or die "Could not get host list"; +print "Getting hostlist from nagios-us\n" if $debug; +my $hostlist2 = `ssh nagios-us grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg` + or die "Could not get host list from nagios (dk)"; +my $hostlist = $hostlist1 . $hostlist2; print "Got list:\n$hostlist\n" if $debug>2; +###### Get list of packages that can be manually maintained +print "getting restricted package versions\n" if $debug; +my %restrpkgs; +my $restplines = `$restrictedpackages` + or die "Could not get the list of restricted packages " . + "from $restrictedpackages: $! "; +print "Got package list: \n$restplines\n" if $debug>2; +my $pname; +my $pver; +for my $pline ( split("\n",$restplines) ) { + chomp($pline); + $pname = $1 if $pline =~ /^Package:\s+(\S*)\s*$/; + $pver = $1 if $pline =~ /^Version:\s+(\S*)\s*$/; + print "$pline: p=$pname v=$pver\n" if $debug>2; + if ( $pname && $pver ) { + print "\nPackage $pname version $pver \n" if $debug>2; + if ( ! $restrpkgs{$pname} ) { + $restrpkgs{$pname} = $pver; + print "found $pname, first version $pver\n" if $debug>1; + } else { + my $bver = $restrpkgs{$pname}; + `dpkg --compare-versions "$bver" lt "$pver" 2>/dev/null `; + if ( ! $? ) { + print "found $pname, better version $pver (better than $bver)\n" + if $debug>1; + $restrpkgs{$pname} = $pver; + } else { + print "found $pname, but version $pver is no better than $bver\n" + if $debug>2; + } + } + $pname = ""; # clear for the next one. + $pver = ""; + } +} + +print "got " . scalar(keys(%restrpkgs)) . " restricted packages\n" if $debug; +if ( $debug >1 ) { + for $pname ( sort (keys(%restrpkgs)) ) { + print " $pname " . $restrpkgs{$pname} . "\n"; + } +} + # Statistics my %summary; -my %sechosts; -my %secpkgs; -my %ownhosts; -my %ownpkgs; -my %normhosts; -my %normpkgs; +my ( %sechosts, %secpkgs ); +my ( %ownhosts, %ownpkgs ); +my ( %manhosts, %manpkgs ); +my ( %normhosts, %normpkgs ); my %okhosts; my %skiphosts; my %allhosts; my $sectot = 0; my $owntot = 0; +my $mantot = 0; my $normtot = 0; +my %updlinks; +my %debversions; + +# Pending modification dates +my %olddates; # Read in from the file +my %newdates; # To be written in the new version of the file +my $datefilename = "aptcheck.data"; +my $dateoldfilename = "aptcheck.old"; +my $thisdate = "*"; # indicates really old stuff +my $warndate; # Older than this will be boldfaced + +if ( -f $datefilename ) { + print "Reading dates from $datefilename\n" if $debug; + open F, $datefilename or die "Could not open date file $datefilename: $!"; + while () { + chop(); + my ($pkg, $date) = split; + next unless $pkg; # skip empty lines + $olddates{$pkg} = $date; + print "Date for '$pkg' is '$date' \n" if $debug; + } + close F; + $thisdate = `date +%F`; + chomp($thisdate); + $warndate = `date +%F -d "30 days ago"` ; ; + chomp($warndate); + print "Dates: now: '$thisdate' warn: '$warndate'\n" if $debug; +} else { + print "No datefile $datefilename found, starting from scratch\n"; +} + my $table = "\n"; -for $hline ( split("\n",$hostlist) ) { +#for my $hline ( split("\n",$hostlist) ) { +for my $hline ( sort( split("\n",$hostlist) ) ) { next unless ( $hline =~ /\/([a-z0-9-]+)\.cfg$/ ); my $H = $1; next if ($H =~ /^commands/ ); next if ($H =~ /^servicegroups/ ); print "Checking $H\n" if $debug; $allhosts{$H}=1; - my $apt = `ssh $H apt-get upgrade -s -o 'Debug::NoLocking=true' `; + my $cmd0 = "cat /etc/debian_version"; + my $cmd1 = "apt-cache -q policy " . join(" ",sort(keys(%restrpkgs))); + my $cmd2 = "apt-get upgrade -s -o 'Debug::NoLocking=true' "; # Note, do not append -qq, we want some output even when nothing to do + print "ssh -q $H \"$cmd0; $cmd1 ; $cmd2 \" 2>/dev/null \n" if ($debug>1); + my $apt = `ssh -q $H "$cmd0; $cmd1 ; $cmd2 " 2>/dev/null`; if ( !$apt ) { $table .= "\n"; $table .= ""; + $det .= ""; + $det .= ""; + $det .= ""; + my $datekey = "$H:$restrname"; + if ( $olddates{$datekey} ) { + $newdates{$datekey} = $olddates{$datekey}; + } else { + $newdates{$datekey} = $thisdate; + } + my $dispdate = $newdates{$datekey}; + # if ( $dispdate lt $warndate ) { + if ( 0 ) { # manual packages don't need to be highlighted + $dispdate = "$dispdate !"; + } + $det .= ""; + $det .= "\n"; + my $key = "$restrname"; + if ( !$summary{$key} ) { + $summary{$key} = ""; + } + $summary{$key} .= "$H "; + } + } + $restrname = ""; # clear for next round + $restrinst = ""; + $restrcand = ""; + } next unless $p =~ /^Inst ([^ ]+) \[([^]]+)\] \(([^ ]+) ([^:]+):/; my ( $pkg,$cur,$new,$src ) = ( $1,$2,$3,$4 ); @@ -68,8 +248,8 @@ for $hline ( split("\n",$hostlist) ) { my $key = $pkg; if ( $src =~ /Security/ ) { $det .= "$pkg (s) "; - $sechosts{$H}=1; - $secpkgs{$pkg}=1; + $sechosts{$H} = 1; + $secpkgs{$pkg} = 1; $secs++; $sectot++; } elsif ( $src =~ /Indexdata/ ) { @@ -93,27 +273,62 @@ for $hline ( split("\n",$hostlist) ) { $det .= " "; $det .= " "; $det .= " "; + my $datekey = "$H:$pkg"; + if ( $olddates{$datekey} ) { + $newdates{$datekey} = $olddates{$datekey}; + } else { + $newdates{$datekey} = $thisdate; + } + my $dispdate = $newdates{$datekey}; + if ( ( $dispdate lt $warndate ) && ( $src =~ /Security/) ) { + $dispdate = "$dispdate !"; + } + $det .= ""; $det .= "\n"; + } - $table .= "\n"; - $table .= "\n"; + $table .= "\n"; + $table .= "\n" + if ( $hostcomments{$H} ); $table .= $det if $pkgs; print "\n$table\n" if $debug>2; - last if $H =~/dart/ && $debug; + last if $H =~/diane/ && $debug; } $table .= "
 
$H (skipped)\n"; @@ -54,11 +173,72 @@ for $hline ( split("\n",$hostlist) ) { next; } print "Got apts for $H: \n$apt\n" if $debug>2; - my $det = ""; + my $det = ""; # detail lines my $pkgs = 0; my $secs = 0; my $own = 0; - for $p ( split("\n",$apt) ) { + my $man = 0; + my $restrname = ""; + my $restrinst = ""; + my $restrcand = ""; + my $debver = 0; + for my $p ( split("\n",$apt) ) { + if ( !$debver ) { # first line + $debver = 1; + $p =~ s/(5[0-9.]+)/$1 LENNY !!!/; + $p =~ s/(6[0-9.]+)/$1 squeeze/; + $p =~ s/(7[0-9.]+)/$1 wheezy/; + $p = " Debian $p"; + $debversions{$H} = $p; + print "Deb version for $H is $p\n" if ($debug>1); + next; + } + # parse apt-cache output + $restrname = $1 if $p =~ /^(\S+):$/; + $restrinst = $1 if $p =~ /^\s+Installed:\s+(\S+)$/; + $restrcand = $1 if $p =~ /^\s+Candidate:\s+(\S+)$/; + if ( $p =~ /^\s+Version table:/ ) { # have all for that package + my $bver = $restrpkgs{$restrname}; + if ( ( $restrinst eq $restrcand ) && + ( $restrinst ne $bver ) ) { + # if different, it is a regular apt upgrade, and will be seen + # later. AND we want to have a different version in our repo + `dpkg --compare-versions "$bver" lt "$restrinst" 2>/dev/null`; + if ( $? ) { # It was not a downgrade + # manual packages may be ahead of the repo! + $mantot++; + $man++; + $pkgs++; + $manhosts{$H} = 1; + $manpkgs{$restrname} = 1; + $det .= "
  $restrname (M)". strdiff($bver,$restrinst)."". strdiff($restrinst,$bver)."" . $dispdate . "
$cur$new" . $dispdate . "
 
$H  \n"; + $table .= "
 
$H  \n"; if ( $pkgs ) { $table .= "$pkgs packages to upgrade. "; $table .= "$secs security. " if $secs; - $table .= " $own from indexdata " if $own; + $table .= " $own from indexdata. " if $own; + $table .= " $man manual. " if $man; } else { $table .= "ok"; $okhosts{$H} = 1; } my $updlink = $wikilink . ucfirst($H) . "Updates" . $year; + # Fix some pages that do not follow the convention. + # Mostly because the host names would not make proper WikiWords + $updlink =~ s/Bugzilla3Updates/BugzillaUpdates/; + $updlink =~ s/Opencontent-solrUpdates/OpenContentSolrUpdates/; + $updlinks{$H} = $updlink; $table .= " Upd"; + $table .= " " . $debversions{$H}; $table .= "
$hostcomments{$H}
\n"; +# Save the date file +if ( ! $debug ) { + `mv -f $datefilename $dateoldfilename`; + open F, ">$datefilename" or die "Could not open date file $datefilename for writing"; + for my $k (sort(keys(%newdates)) ) { + print F "$k " . $newdates{$k}. "\n"; + print "date for '$k' '" . $newdates{$k}. "'\n" if $debug; + } + close F + or die "Could not close date file $datefilename: $!"; +} else { + print "Not updating the date file, this is a debug run\n"; +} + # Page header my $outfile = "/tmp/aptcheck.html"; open F, ">$outfile" @@ -122,6 +337,11 @@ print F "\n"; print F "Apt upgrade status\n"; print F "\n"; print F "

Apt package status

\n"; +print F "" . ( $sectot + $owntot + $mantot + $normtot ) . + " packages pending ($sectot critical) \n"; + +print F "

Debug run, many hosts missing!

\n" + if $debug; # Summary table: one row for per host group @@ -133,14 +353,15 @@ print F "Packages\n"; if ( $sectot ) { print F "Security
" . scalar(keys(%sechosts)) . - " / " . scalar(keys(%secpkgs)) . " / $sectot \n" ; + " / " . scalar(keys(%secpkgs)) . " / $sectot \n" ; print F ""; - for $HH ( sort(keys(%sechosts)) ) { - print F "$HH "; + for my $HH ( sort(keys(%sechosts)) ) { + my $upd = $updlinks{$HH} || "#" ; + print F "$HH$updlink "; } print F ""; print F ""; - for $PP ( sort(keys(%secpkgs)) ) { + for my $PP ( sort(keys(%secpkgs)) ) { print F "$PP "; } print F ""; @@ -148,48 +369,99 @@ if ( $sectot ) { } if ( $owntot ) { print F "Indexdata
" . scalar(keys(%ownhosts)) . - " / " . scalar(keys(%ownpkgs)) . " / $owntot \n" ; + " / " . scalar(keys(%ownpkgs)) . " / $owntot \n" ; + print F ""; + for my $HH ( sort(keys(%ownhosts)) ) { + my $upd = $updlinks{$HH} || "#" ; + print F "$HH$updlink "; + #print F "$HH "; + } + print F ""; + print F ""; + for my $PP ( sort(keys(%ownpkgs)) ) { + print F "$PP "; + } + print F ""; + print F "\n"; +} +if ( $mantot ) { + print F "Manual
" . scalar(keys(%manhosts)) . + " / " . scalar(keys(%manpkgs)) . " / $mantot \n" ; print F ""; - for $HH ( sort(keys(%ownhosts)) ) { - print F "$HH "; + for my $HH ( sort(keys(%manhosts)) ) { + my $upd = $updlinks{$HH} || "#" ; + print F "$HH$updlink "; + #print F "$HH "; } print F ""; print F ""; - for $PP ( sort(keys(%ownpkgs)) ) { + for my $PP ( sort(keys(%manpkgs)) ) { print F "$PP "; } print F ""; - #print F "" . join(" ",sort(keys(%ownpkgs))) . " "; print F "\n"; } if ( $normtot ) { - print F "Indexdata
" . scalar(keys(%normhosts)) . - " / " . scalar(keys(%normpkgs)) . " / $normtot \n" ; + print F "Normal
" . scalar(keys(%normhosts)) . + " / " . scalar(keys(%normpkgs)) . " / $normtot \n" ; + print F ""; + for my $HH ( sort(keys(%normhosts)) ) { + my $upd = $updlinks{$HH} || "#" ; + print F "$HH$updlink "; + #print F "$HH "; + } + print F ""; print F ""; - for $HH ( sort(keys(%normhosts)) ) { - print F "$HH "; + for my $PP ( sort(keys(%normpkgs)) ) { + print F "$PP "; } - print F "" . join(" ",sort(keys(%normpkgs))) . " "; + print F ""; print F "\n"; } if ( %skiphosts ) { - print F "Skipped: " . scalar(keys(%skiphosts)) . "\n"; + print F "Skipped " . scalar(keys(%skiphosts)) . "\n"; print F ""; - for $HH ( sort(keys(%skiphosts)) ) { - print F "$HH "; + for my $HH ( sort(keys(%skiphosts)) ) { + my $upd = $updlinks{$HH} || + $wikilink . ucfirst($HH) . "Updates" . $year; + print F "$HH$updlink "; + #print F "$HH "; } print F "\n"; } -if ( %okhosts ) { - print F "Ok: " . scalar(keys(%okhosts)) . "\n"; +if ( 1 ) { + print F "Ok
" . scalar(keys(%okhosts)) . + " of " . scalar(keys(%allhosts)) . "\n"; print F ""; - for $HH ( sort(keys(%okhosts)) ) { - print F "$HH "; + for my $HH ( sort(keys(%okhosts)) ) { + my $upd = $updlinks{$HH} || "#" ; + print F "$HH$updlink "; + #print F "$HH "; + } + if ( !%okhosts ) { + print F "None at all!"; } print F "\n"; } print F "\n"; +print F "

" . ( $sectot + $owntot + $mantot + $normtot ) . + " packages pending ($sectot critical) \n"; + +# Graph +#my $secs = 60*60*24 * 7 * 2; # 2 weeks in secods +#my $secs = "1m"; # one month, let nagios do the math +my $secs = "45d"; +print F "

" . + "\n". + " ". + "" . + "
\n"; # The host table print F $table; @@ -197,13 +469,14 @@ print F $table; # Package table print F "

Packages\n"; print F "\n"; -for $P ( sort(keys(%summary)) ) { +for my $P ( sort(keys(%summary)) ) { my $PN = $P; - $PN = "$P (s)" if ($secpkgs{$P}); - $PN = "$P (id)" if ($ownpkgs{$P}); + $PN = "$P (s)" if ($secpkgs{$P}); + $PN = "$P (id)" if ($ownpkgs{$P}); + $PN = "$P (M)" if ($manpkgs{$P}); print F "\n"; print F "\n"; @@ -228,14 +501,17 @@ exit(0); sub strdiff { my $x = shift; my $y = shift; - print "strdiff: '$x' '$y' \n" if $debug>1; + print "strdiff: '$x' '$y' \n" if $debug>2; + if ( $x eq $y ) { + return "$x ??"; + } my $a = 0; while ( $a < length($y) && substr($x,$a,1) eq substr($y,$a,1) ) { $a++; } if ( $a == length($y) ) { - return "$y ???"; + return "$y"; } my $b = 1; while ( $b < length($y)-$a && @@ -243,14 +519,14 @@ sub strdiff { $b++; } my $c = length($y) - $b +1; - print "strdiff: a=$a " . substr($y,0,$a) ."\n" if $debug>1; - print "strdiff: b=$b " . "\n" if $debug>1; - print "strdiff: c=$c " . substr($y,$c) ."\n" if $debug>1; - print "strdiff: " . substr($y,$a, $c-$a) ."\n" if $debug>1; + print "strdiff: a=$a " . substr($y,0,$a) ."\n" if $debug>2; + print "strdiff: b=$b " . "\n" if $debug>2; + print "strdiff: c=$c " . substr($y,$c) ."\n" if $debug>2; + print "strdiff: " . substr($y,$a, $c-$a) ."\n" if $debug>2; my $z = substr($y,0,$a) . "" . substr($y,$a, $c-$a) . "" . substr($y,$c); - print "strdiff: " . $z ."\n" if $debug>1; - print "\n" if $debug>1; + print "strdiff: " . $z ."\n" if $debug>2; + print "\n" if $debug>2; return $z; }
$PN"; - for $HH ( split(' ',$summary{$P} )) { + for my $HH ( split(' ',$summary{$P} )) { print F "$HH "; } print F "