#!/bin/bash
## Given an HTML file, check for broken and permanently-moved links -- by Eugene Reimer 2010-04-25;
## curdir must be where the input file resides, if it contains unqualified links (HREFs);
## leading-slashed link is resolved wrt optional 2nd param (HOME), default: first two components of curdir name;
##
## PREREQ:  fullname  fullnameNOSLASH -- from http://ereimer.net/programs/general-purpose-scripts.htm
##
## USAGE EXAMPLE:
##   for H in *{htm,html};do WEBLINKCHECK-LL $H $WEBHOME; done	##for each HTML file, check its links  (see weblinkcheck.cron for a complete example)
##
## Copyright © 2010 Eugene Reimer;  can be used, modified, copied, and distributed or sold under the terms of either the LGPL or the GPL (your choice);
## see http://www.gnu.org/licenses for the details of these terms.

if [ $# -ge 2 ];then  HOME=$2						##set HOME from 2nd param
else HOME=$(pwd); while [[ $HOME == /*/*/* ]];do HOME=${HOME%/*};done	##set HOME to the first two components of curdir name
fi
USERAGENT="Opera/9.63 (X11; Linux i686; U; en) Presto/2.1.1"				##UserAgent-string for Opera on Linux
WGET () { wget -U"$USERAGENT" --tries=5 --timeout=120 --no-check-certificate "$@"; }	##invoke wget with more moderate tries & timeout settings;  supply UserAgent

cat "$1"						|
sed $'s|-->|\x02|g'					|	##replace comment-ender with x02
sed $':a $!{N;ba}; s|<!--[^\x02]*\x02||g'		|	##discard comments, treating file as one line
sed $'s|\x02|-->|g'					|	##x02 back to comment-ender for unmatched uses of "-->"
cat >/tmp/WEBLINK-tmp						##comment-stripped copy of input-file to tmpfile
BASE=$(cat /tmp/WEBLINK-tmp |grep -i '<base.*href=' |sed "s|.*<[Bb].*[Ff]=\([^ \t>]*\).*|\1|; s|['\"]||g")	##BASE gets base-href if present, emptystring if not

cat /tmp/WEBLINK-tmp					|
grep -i '<a.*href='					|	##isolate lines containing <a.*href=  (case-blindly)
sed 's|[Hh][Rr][Ee][Ff]=|\n\L&|g'			|	##ensure at most one href= per line, also standardize to lowercase
sed -n 's|href=\([^ \t>]*\).*|\1|p'			|	##isolate just the URL part of each href=URL
sed "s|['\"]||g; s|#.*||"				|	##remove quotes and strophs;  remove any within-page link-component (can result in emptystring)
sort -u							|	##uniquify
egrep -iv 'mailto:|javascript:|^$'			|	##ignore mailto, javascript, emptystring links
while read;do
  F=$REPLY;  [[ $F != *://* && $F != /* ]] && F=$BASE$F		##handle BASE for unqualified ref the way Apache does
  if [[ $F == *://* ]];then					##==http:  use wget to test for existence  (also handle ftp https?)
    MSG="/tmp/WEBLINK-msg-$(fullnameNOSLASH $1==$F)"		##name of wget-msgs tmpfile, to be kept for anomalous case
    WGET -O/tmp/WEBLINK-tmp2 $F &>"$MSG";  RC=$?		##wget with msgs to tmpfile for subsequent scanning
    GETRC=$(cat "$MSG" |sed -n '/HTTP request sent/s|.* \([0-9]\+\).*|\1|p' |tail -n1)				##get the HTTP-RC from wget-msgs
    [[ $RC -eq 0 ]] || echo "$(fullname $1) $F ==BROKEN-$GETRC"	##msg for broken-link, RC is always 1 for failure, but the HTTP-RC can be of interest
    [[ $RC -eq 0 && $GETRC -ne 200 ]] && echo "$(fullname $1) $F ==WGET-ANOMALY"				##never happens?
    [[ $RC -eq 0 ]] || continue					##skip looking at 301-redirect if redirected-result still broken, and keep $MSG for these
    NEW=$(cat "$MSG" |sed -n '/301 Moved Permanently/{N;N;s|.*\nLocation: ||;s| .*||;p}' |tail -n1)		##get 301-redirect-Location from wget-msgs, if any
    [[ $NEW != "" ]] && echo "$(fullname $1) $F ==MOVEDTO:$NEW"	##msg for 301-redirect
    [[ $RC -eq 0 && $GETRC -eq 200 ]] && rm -f "$MSG"		##discard $MSG file except for anomalous case
  else								##==local file:
    [[ $F == /* ]] && F=$HOME$F					##handle leading-slash the way Apache does
    [[ $HOME == /er* && $F == *[0-9][0-9][0-9].htm && ! -e $F ]] && F=${F/.htm/.jpg}				##==simulate Apache-rewriterule from .htaccess (KLUGE)
    [[ -e $F ]] || echo "$(fullname $1) $F ==NOTFOUND"		##msg for broken-link
  fi
done


exit
NOTE: doesn't properly parse HTML;  the comment-skipping and HREF-matching will only work if "<!--", "-->", "href=" are avoided in quoted-strings and javascript;
handles "<BASE HREF=";
the link-checking matches "<A HREF=" (case-blindly), thus skipping HREF in BASE-tag and in "&lt;a...href=...&gt;" as used when HTML-examples are written in HTML;

reports 301-redirect so such link can be revised to the new name / location  (ignores 302-redirect);
	involves scanning $MSG:  the line after its last "301 Moved Permanently" line gives the final Location, except it's sometimes 2nd line after, 
	hence the handling of 3-line Moved Permanently...Location;
	eg: wget http://adobe.com/go/getflashplayer -- produces 3 consecutive 301-redirects followed by a 302-redirect (the 301's suggest brain-damage at adobe:)

wget's timeout for nonexistent website took way too long!!  (eg http://biodiversity.uno.edu/...);
	see its man-page for:  --tries=number  --dns-timeout=seconds  --connect-timeout=seconds  --read-timeout=seconds  --timeout=seconds (all 3);
	defaults:   --tries=20 --read-timeout=900	==> worst case was 20 * 15min = 5-hours!!
	switch to:  --tries=5       --timeout=120	==> worst case now  5 *  2min = 10-min;
	--worst case is actually worse than that, eg: wget times-out while connecting, retries & succeeds, then times-out in read, for total of 4+ minutes on one "try";

was getting 403 (Forbidden) on some URLs that work in a browser (eg http://celebrity.myheritage.com/celebrity-morph);
	(A) USERAGENT=Mozilla							-- solves 4 failures, but brings 5 new ones:
	(B) USERAGENT="Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)"	-- solves same 4, brings 3 new ones, adobe-getflashplayer gets worse, is erratic:
	(C) USERAGENT="Opera/9.63 (X11; Linux i686; U; en) Presto/2.1.1"	-- solves same 4, and solves adobe-getflashplayer;  ==SEEMS THE BEST ANSWER:
	--several of the "solutions" are of dubious merit, producing pages of the domain-name-for-sale sort which I'd rather have seen as failures;
	--turns out the adobe-getflashplayer redirects remain weirdly erratic with every USERAGENT I've tried, and may have to ignore them to avoid "thrashing";

==NOTE: could treat <FORM ACTION= as links, in addition to <A HREF=, although not crucial for my use...

==NOTE: could avoid RewriteRule-KLUGES by using Apache (via wget) for local references too??
	my use needs such kluge for er-site's htm-to-jpg rule, but may need more for other sites especially Drupal sites;
	(would use something like:  wget http://$PROJ/$FDIR_LESS_$HOME/$F...)

2010-05-06: now keep tmpfile of wget-msgs for anomalous case, named in the fullnameNOSLASH-style, for later review;
2010-11-27:  added --no-check-certificate to wget-options, in WGET function, to fix problem described in weblinkcheck.cron;