#!/bin/bash
#
# Copyright Hank Leininger <hlein@korelogic.com>, <hlein@progressive-comp.com>

VERSION='$Id: www-version-harvest,v 1.2 2005/05/10 22:31:23 hlein Exp $'
VERSION=`echo "$VERSION" | sed 's/.*,v //; s/ .*//;'`

# given a URL,
#   go get it
#   find webservers it points to
#   for each server,
#     grab webserver version
#     or print 'NONE' for any error
#
# Requires perl with a working Socket module, netcat, and
# at least one of curl, wget, or lynx in PATH.
#
# Primarily useful for verifying webserver versions on mirror
# sites--the higher the percentage of old, vulnerable Apache
# and whatnot, the less willing you should be to trust any
# package from any of those mirrors without signatures and
# possibly still, code- and diff- review.
#
# XXX: Should we do any post-processing of the results?
#      Flagging which ones look vulnerable, etc?
#      I'm inclined not to; too much to maintain.

trap 'exit 1' SIGINT SIGTERM SIGPIPE

URL="$1"
export DEBUG=1

BASENAME=`echo "$0" | sed 's%.*/%%'`

if [ -z "$URL" -o "$URL" = "-h" ]; then
  echo "Usage: $BASENAME start_URL" 1>&2
  echo "$BASENAME v$VERSION" 1>&2
  exit 1
fi

hash nc 2>/dev/null || { echo "Need nc in my path; quitting." 1>&2; exit 1; }

# XXX: should fall back to netcat but I'm lazy++
{ hash curl 2>/dev/null && GET="curl -s -- "; } || \
{ hash wget 2>/dev/null && GET="wget -q -O - -- "; } || \
{ hash lynx 2>/dev/null && GET="lynx -source "; } || {
	echo "Need curl, wget, or lynx and you have none; quitting." 1>&2
	exit 1
}

for SITE in `{ echo "$URL"; $GET "$URL"; } | \
perl -MSocket -ne 'chomp; while
	(m%(?:value="?|href="?|^\s*)(?:https?|ftp)://([^:/>"\s]+)%gi) { 
  $s=$1; $sites{$s}++; $sites{$s}++ if ($s=~s/^ftp/www/);
} END { foreach my $site (keys %sites) {
    print "$site\n" if ($ENV{DEBUG});
    my @addrs=gethostbyname($site);
    unless (@addrs) { warn "gethostbyname($site) failed; skipping: $!\n"; next }
    @addrs=map { inet_ntoa($_) } @addrs[4..$#addrs];
    if (scalar(@addrs) == 1) { $siteips{$site}++ }
    else { foreach (@addrs) { $siteips{ $site eq $_ ? $site : "$site|$_" }++ }}}
  print join ("\n", sort keys %siteips), "\n" }'` ; do

  IP=`echo "$SITE" | cut -d\| -f2-`
  # XXX: 39 is awfully wide, but for hostnames with multiple IPs, it's
  #      needed to look reasonable.  Perhaps perl should spit out hints
  #      about the format (longest site:ip is...)
  printf "%-39s " "$SITE"
  echo -e "HEAD / HTTP/1.0\r\n\r\n" | \
	nc -w 6 "$IP" 80 2>/dev/null | \
	egrep "^Server: " || \
	echo "NONE"
done