#!/bin/bash
## RECEIVES:  $1 one of: /{er,debwendon,noci}/website;  (until 2008-11-01: curdir was one of /{er,debwendon,noci}; and $1 was website|reimer2|reimer3|dotnet|...)
##	      $2 (optional) name of SUBDIR for uploading it and only it;
##
## PREREQ:  find-dirs -- from http://ereimer.net/programs/general-purpose-scripts.htm
##
## see also: WEBGET, WEBGETLOGS;
##
## BEWARE: avoid SEMICOLON, AMPERSAND, COMMA  in lftp-comments  (as an unexpected "feature" it treats those as end-of-comment, altho COMMA is even weirder)
## BEWARE: the lftp-cd-command is not allowed to have a comment after it--!!--
## NOTE: the NON-STD indentation within HERE-DOCUMENT is essential--!!--  because TAB(s) needed before ENDOFHERE, whereas lftp likely handles spaces before verb?
##
## USAGE EXAMPLES:
##	WEBPUT-LL  /noci/website			<--from webput, for /noci
##	WEBPUT-LL  /noci/website  wa-stats		<--from webalize-auto to upload nothing but wa-stats dir for /noci;  NOTE: only this kind does DOWNLOADSTATS!!

source  /etc/sbin/WEBVARS						##define FTP USERID PASSWORD ROOT DOM URL LOCURL
HOST=${URL#http://};        HOSTA=${HOST//./\\\\.}			##HOSTA     for .htaccess fixups eg: ereimer\.net  is matched by sed-pattern ereimer\\.net
LOCHOST=${LOCURL#http://};  LOCHOSTA="${LOCHOST}\\\$"			##LOCHOSTA  for .htaccess fixups eg: er$           is matched by sed-pattern er\$
##echo -n "CWD:$(pwd) HOSTA=$HOSTA; LOCHOSTA=$LOCHOSTA; Continue?";read	##debug
echo "WEBPUT $(date +%Y-%m-%d-%H:%M): dir=$1 ftp=$FTP userid=$USERID remote-dir=$ROOT"
##set -v
##set -x
##clean $1								##yanked, now the caller's responsibility
cd $1  || exit 9							##==LOCAL cd for DIRS-loop;  rest of code has fully-qualified names (was for curdir==/er|/noci)
PreHtmFixup()  {							##==PreHtmFixup: receives list of HTM-filenames
  chgsed -k -n -q '/alt="/!s|.title=\("[^"]*"\)|& alt=\1|g'	"$@"	##add ALT, by copying TITLE, on lines lacking ALT -- w/o backup
}
PostHtmFixup() {							##==PostHtmFixup: receives list of HTM-filenames
  chgsed -k -n -q '/title="/s|.alt="[^"]*"||g'			"$@"	##remove ALT on lines having TITLE -- w/o backup
}
PreHtaFixup()  {							##==PreHtaFixup: receives HTACCESS-filename
  [ -e $1 ] || return
  chgsed -k -n -q "s|$LOCURL|$URL|g; s|$LOCHOSTA|$HOSTA|g"	"$1"	##eg:  http://er-->http://ereimer.net  er$-->ereimer\.net
}
PostHtaFixup() {							##==PostHtaFixup: receives HTACCESS-filename
  [ -e $1 ] || return
  chgsed -k -n -q "s|$URL|$LOCURL|g; s|$HOSTA|$LOCHOSTA|g"	"$1"	##eg:  http://ereimer.net-->http://er  ereimer\.net-->er$
}

DOWNLOADSTATS=Y;  [[ $2 == "" ]] && DOWNLOADSTATS=;	##2006nov:YANKED, hourly STATSGET via crontab;  2008-10:UNYANKED;  2009-12:skip if $2 omitted
if [ $DOWNLOADSTATS ];then				##==<<Download-Stats>>==  2008-10: removed test for lowest-dir being website
	echo "WEBPUT-GETSTATS to $1/webalizer:"		##at HostExcellence, preserve webalizer subdir (access-logs are separate)
	lftp <<- ENDOFHERE
	open -u $USERID,$PASSWORD  $FTP
	mirror -vvv  $ROOT/webalizer  $1/webalizer	##download  2008-10-11 stats-to-webalizer subdir
	quit
	ENDOFHERE
fi
##-----do queued-renames, if there are any;  2009-12-08: skip if $2 specified:
if [[ $2 == "" ]] && [ -s $1-queue ];then
	echo "WEBPUT-CMDS applying $1-queue renames:"
	lftp <<- ENDOFHERE
	open -u $USERID,$PASSWORD  $FTP
	cd $ROOT
	source $1-queue					##do remote renaming - see webmv script - needs remote cd $ROOT
	quit
	ENDOFHERE
	{ echo "##--WEBPUT--$(date +%y%m%d-%H%M)--";  cat $1-queue; }  >>$1-RENAMES		##keep a log of such renames
	>$1-queue										##make the queue empty
fi
##-----do the subdirs:
DIRS=$2;  [[ $2 == "" ]] && DIRS="$(find-dirs) ."	##2009-12-08: added "." to combine root-dir with find-dirs loop, in order to skip "." when $2 specified
for DIR in $DIRS;do					##for each dir (doing root-dir last if at all)...
    if [[ $DIR == "photo-upload" ]];then		##==do photo-upload directory;  2010-12-15:added this special-case
	if OLDER $DIR $1-ls;then continue; fi										##skip if olderthan $1-ls, quietly
        cp -f $1/$DIR/{.ht*,*cgi}  /tmp/$DIR										##make backup, in case of clobbering
	echo "WEBPUT-DOWNLOAD to $1/$DIR:"
	lftp <<- ENDOFHERE
	open -u $USERID,$PASSWORD  $FTP
	set ftp:list-empty-ok yes
	cd $ROOT
	mirror -vvv -n  $DIR  $1/$DIR											##download WITHOUT deletion
	quit
	ENDOFHERE
	echo "WEBPUT-UPLOAD from $1/$DIR W/O DELETION:"
	chgsed -k -n -q '/^AuthUserFile/s|/er/website|/hsphere/local/home/ereimer/ereimer.net|g'  $1/$DIR/.htaccess	##PRE-upload-fixups to photo-upload/.htaccess
	lftp <<- ENDOFHERE
	open -u $USERID,$PASSWORD  $FTP
	set ftp:list-empty-ok yes
	cd $ROOT
	mirror -vvv -Rrn  $1/$DIR  $DIR											##upload WITHOUT -e (deletion) - BEWARE BROKEN
	quit
	ENDOFHERE
	chgsed -k -n -q '/^AuthUserFile/s|/hsphere/local/home/ereimer/ereimer.net|/er/website|g'  $1/$DIR/.htaccess	##POST-upload-fixups to photo-upload/.htaccess
    elif [[ $DIR != "." ]];then				##==do non-root non-photo-upload directory
	if [[ $DIR == webalizer ]];then continue; fi	##skip webalizer dir, quietly
	if OLDER $DIR $1-ls;then continue; fi		##skip dir olderthan $1-ls, quietly;  2009-08-31: OLDER/NEWER for dir-workaround;  was -ot
	echo "WEBPUT-UPLOAD from $1/$DIR:"
	PreHtmFixup $1/$DIR/*htm			##apply PRE-upload-HTM-fixups
	lftp <<- ENDOFHERE
	open -u $USERID,$PASSWORD  $FTP
	set ftp:list-empty-ok yes			##2008-03-01: fixes the reconnect-woes for subdirs
	cd $ROOT
	mirror -vvv -Rrne  $1/$DIR  $DIR		##upload -- 2008-10-26: using -r (no-recursion) -n (only-newer) and -e (with-deletion) -- was w/o delete
	quit
	ENDOFHERE
	PostHtmFixup $1/$DIR/*htm			##apply POST-upload-HTM-fixups
    else						##==do root-dir + produce $1-ls listing
	echo "WEBPUT-UPLOAD from $1:"
	PreHtmFixup $1/*htm; PreHtaFixup $1/.htaccess	##apply PRE-upload-HTM+HTACCESS-fixups
	lftp <<- ENDOFHERE
	open -u $USERID,$PASSWORD  $FTP
	set ftp:list-empty-ok yes			##2008-03-01: fixes the reconnect-woes for subdirs
	cd $ROOT
	mirror -vvv -Rrne  $1  .			##upload -- 2008-10-26: using -r (no-recursion) -n (only-newer) and -e (with-deletion) -- was w/o delete
	echo "WEBPUT-LS to $1-ls:"
	! [ -e $1-ls ] && mv -f $1-ls  $1-ls-before	##for before and after timestamps, sizes
	ls -AR . >$1-ls					##==download $1-ls -- works at NewWinnipeg + at MTS after 06nov hosting change + at HostExcellence
	cls -l . >$1-cls				##==TEMP to investigate timestamp/size/permissions quirks after 06nov hosting change??
	quit
	ENDOFHERE
	PostHtmFixup $1/*htm; PostHtaFixup $1/.htaccess	##apply POST-upload-HTM+HTACCESS-fixups
	##=====NEEDED: ping google et al about newly revised sitemap (see notes in sitemap-gen):
	##wget  www.google.com/webmasters/tools/ping?sitemap=http://ereimer.net/sitemap.xml	##note: need $URL ==> best to combine WEBURL & WEBVARS!!
	##rm -f  ping*										##wget creates such;  it has confirmation from google;  safely discarded
    fi
done



exit
2004dec: was getting truncated <ls> output, due to limitation at MTS;  solved by doing <ls> in two parts:  [0-9A-R]*  [S-Za-z]*
		ls -AR [0-9A-R]* >"$1"-ls		##for old MTS, needed 2 steps of <1000 filenames each
		ls -AR [S-Za-z]* >>"$1"-ls		##for old MTS, needed 2 steps of <1000 filenames each
	will someday need more steps, since seem to be limited to roughly a thousand lines of ls-output, at MTS?
	2005aug31:  Yup, the limit was imposed by MTS -- using lftp at NewWinnipeg works as expected, so was back to single ls (briefly);
	2006nov:    also OK after MTS-outsourcing, so back to single ls;

MTS-STATS and lftp:
====================
2005feb01 thru 2005mar16, was downloading MTS-stats to  /noci/website-stats  (now /pix/WEBSTATS-noci);
2005mar17: changed to  /noci/website/stats  -- so that "clean" can be done again.
	-- (the monthly stats will continue to reside under website-stats, now WEBSTATS-noci)
	-- (they must be separate, so that the "mirror" for uploads doesn't upload them!)

Downloading the stats may be essential during a "WITH DELETION", to avoid losing stats--??--
	am doing it on every "webput", to be safer, and because that was useful while learning how MTS did things...
Doing a "webput" near 4:30am may be dangerous - since that is when MTS appends previous-day's data to it.
Question: having an "old" website/stats/ftp151jjaccess_log file may be perfectly safe, with or without Deletion,
	since lftp-mirroring shouldn't replace a newer file with an older--??--
	-- at least not if I use the -n (only-newer) option to lftp's mirror cmd
	-- 2005aug31: started using -r (no recusion to subdirs) option to lftp's mirror cmd - this surely makes the stats-download useless??
	   (but I'm used to having the stats handy after webput, in case I feel like looking at 404's etc)

NOTE: if a first-of-month GET/ROTATE has failed or not been done for some reason, then i need to:
	first ensure that i have a local copy with enough data, rerunning WEBLOGSROTATE (or WEBPUT-LL), if needed,
	then manually edit /noci/website/stats/ftp151jjaccess_log -- making it as it should be remotely,
	then run WEBPUTSTATS.  (was: run WEBPUT-LL, but with Download-Stats step yanked AND with "-r" REMOVED from the mirror cmds to lftp)
	2006aug02: couldn't get stats-UPload to work, removing -r, adding -n, adding --ignore-size, all no good??  ==made <WEBPUTSTATS> is faster & works!!

NEW-WINNIPEG WEBHOSTING (BRIEF):
2005aug31:  changed . to www  -- for NW (NewWinnipeg)
	then to notwanted-www
	then to native-www.  
	then to nativeorchid-www.  
2005aug31:  started using -r on mirror, to be safer WRT files under the stats directory
TEMP, to get redirect-files to NewWinnipeg  (BETTER to use 301-redirects in .htaccess), used the following:
	echo "WEBPUT: uploading $1-redirects";  mirror -v -Rr  "$1"-redirects .	##TEMP upload redirect files to NewWinnipeg

MTS-HOSTING changes:
2006oct17: (cd /noci; webput) failed with "Login failed: 530 Login incorrect"
2006oct20: using New MTS instructions:  get infinite (reconnect) retrying;
2006nov09: after the move to the new webhoster, lftp repeats all recent uploads - not sure why, but must be timestamp-related;
	Examples of unwanted repetition:  identical size but remote-ts newer than mine  (remote-ts 1-hr newer than time of last upload);
 	tried   --newer-than=website-ls  - works but lessens the beauty of mirror  (is still better than a simple script wrt screwups at hoster...)
	trying  -n (only-newer) 

TIMESTAMP ISSUES (2006dec):
(1) while studying access_logs, observed that a file downloaded by lftp-mirror has a timestamp FIVE-hours earlier than it should be;
	e.g.  11:23  when it should be  16:23;
	theory: lftp detects their computer is on EST, but asuumes its clock is on GMT, and so subtracts 5;
	(not very satisfying since lftp should be correcting to my computer's time which is -6 in wintertime?)
(2) comparing lftp's ls and cls output:
	ran /noci/webput on 2006-Dec02 at 04:34, and it uploaded UNINDEXED-DonateButtons.htm,
	its cls-output:  Dec  1 23:34 ./UNINDEXED-DonateButtons.htm  <-- has the FIVE-hours-in-the-past problem!
	its ls-output:   Dec  2 05:34 UNINDEXED-DonateButtons.htm    <-- has a ONE-hour-in-future problem instead (is correct EST time)
2006dec04: upgraded lftp to 3.2.1 (newest for FC4 via yum) - have very little hope it will change anything timestamp-related.

NOTES THAT WERE ALONGSIDE THE LFTP-MIRROR-CMD (the WITHOUT-DELETION one):
2006nov: first tried --newer-than=website-ls;  then went to -n (only-newer) instead;
2007mar30: subdirs not working, on uploads to HostExcellence-ereimer.net site--??--
2007mar30: seems one must wait a while between mkdir & using it--??--
2007apr25: revised htm files under 2002, 2003, 2004, did NOT get uploaded 
	unyanking subdir-lines below fixes ==> absence of -r does NOT behave as documented--??--
	unyanking subdir-lines below fixes ==> seem to need separate scripts for ER & NOCI==!!==
2007dec08:  subdir shopping_file under subdir website/NOPERCART-DEMO3 did get mirrorred -- a further mystery--!!--
2008-03-01:  on NEW HostExcellence, back to subdirs not working  (tried -R instead of -Rn, but made no difference)
2008-03-01:  on OLD HostExcellence, same problem on DEMO4 (it's new there too) -- but lftp eventually went on
		suggests NOT creating subdirs with mkdir works better than creating them??
2008-03-01:  ==AHA an answer to my subdir woes (http://www.mail-archive.com/lftp@uniyar.ac.ru/msg01666.html):
		set ftp:list-empty-ok yes    (inline or in /etc/lftp.conf)
2008-03-03:  now getting "Delaying before reconnect" on the "cd" command up front--??--
		need to make those delays shorter  ==net:reconnect-interval-max  reduced in /etc/lftp.conf==
2008-03-04:  more "Delaying before reconnect" problems:
		with -Rn with or without list-empty-ok  then it happens on the mirror cmd - unending;
		with -R  without list-empty-ok  then it happens on the cd cmd - unending;
		with -R  and list-empty-ok  then it ends eventually on the cd,  but get it on mirror - unending;
		eventually ended after about 12-hours;  after that uploads were fine;  some glitch at HostExcellence??
2008-03-05:  a timestamp-issue: 
		debwendon index.html was Mar 5 14:31 (or 20:31 in GMT);  their system (ls) shows it as Mar 6 04:31
		suggests their system is GMT+08 ??
		--their clock-screwup went away soon after I asked HostExcellence about it...
2008-10-26:  went back to -r and -n on the mirror-cmd;

AND HERE ARE THE MANUAL-SUBDIR-MIRRORING CMDS:
rror -vvv -R --newer-than=website-ls  $1  .	##upload w/o delete  ==06nov: tried newer-than...
mirror -vvv -Rrn                        $1  .	##upload w/o delete  ==06nov: tried -n (only-newer) instead?  ==07mar30: removed -r  ==08oct26: back to -r and -n
echo "DEMO2";  mirror -vvv -Rrn website/NOPERCART-DEMO2  NOPERCART-DEMO2	##--07nov20: TEMP upload files in subdirs (ER)
echo "DEMO3";  mirror -vvv -Rrn website/NOPERCART-DEMO3  NOPERCART-DEMO3	##--07dec04: TEMP upload files in subdirs (ER)
echo "DEMO4";  mirror -vvv -Rrn website/NOPERCART-DEMO4  NOPERCART-DEMO4	##--08mar01: TEMP upload files in subdirs (ER)
echo "DEMOC";  mirror -vvv -Rrn website/NOPERCART-DEMOC  NOPERCART-DEMOC	##--08mar06: TEMP upload files in subdirs (ER)  08mar08 DEMO-COUPON-->DEMOC
echo "DEMORB"; mirror -vvv -Rrn website/NOPERCART-DEMORB NOPERCART-DEMORB	##--08mar06: TEMP upload files in subdirs (ER)  08oct21 for Roger Britt
echo "DEMOD";  mirror -vvv -Rrn website/NOPERCART-DEMOD  NOPERCART-DEMOD	##--08feb18: TEMP upload files in subdirs (ER)  08mar08 discarded DEMO-DEBUG  OBSOLETE, KEEP YANKED
echo "DEMO5";  mirror -vvv -Rrn website/NOPERCART-DEMO5  NOPERCART-DEMO5	##--08jan20: TEMP upload files in subdirs (ER)  for barkaparka.com.au         OBSOLETE, KEEP YANKED
echo "DEMO6";  mirror -vvv -Rrn website/NOPERCART-DEMO6  NOPERCART-DEMO6	##--08jan20: TEMP upload files in subdirs (ER)  for barkaparka.com.au OBSOLETE 2008-11-03, one week after 2008-10-27
echo "2002";   mirror -vvv -Rrn website/2002		    2002		##--07mar30: TEMP upload files in subdirs (ER)
echo "2003";   mirror -vvv -Rrn website/2003		    2003		##--07mar30: TEMP upload files in subdirs (ER)
echo "2004";   mirror -vvv -Rrn website/2004		    2004		##--07mar30: TEMP upload files in subdirs (ER)
echo "2005";   mirror -vvv -Rrn website/2005		    2005		##--07mar30: TEMP upload files in subdirs (ER)
echo "2006";   mirror -vvv -Rrn website/2006		    2006		##--07mar30: TEMP upload files in subdirs (ER)
echo "KPLR";   mirror -vvv -Rrn website/KPLR		    KPLR		##--07mar30: TEMP upload files in subdirs (ER)
echo "Sid";    mirror -vvv -Rrn website/Sid		    Sid 		##--07nov12: TEMP upload files in subdirs (ER)
echo "cgi-bin";mirror -vvv -Rrn $1/cgi-bin		    cgi-bin		##--08mar13: TEMP upload files in subdirs (ER / Nativeorchid / Debwendon)


2008-10..2008-11-01: Rewrote for HostExcellence:  (1) handle subdirs;  (2) always use With-Deletion;  (3) switch to redirects via .htaccess;
(1)  Handle subdirs:
	traverse subdirs of $1 using:  for DIR in $(find-dirs)...   <-- much like: for DIR in $(find -type d|sort)...
	revised all local-file-references to be fully-qualified ($1-->$CWD/$1),  so can run with different curdir;
	then revised calling-interface:  $1 is fully-qualified, one of /er/website /debwendon/website /noci/website;  then $CWD/$1-->$1;
(2)	PreHtmFixup, PostHtmFixup: because the TITLE+ALT-fixups are needed on each subdir, now do them here (rather than in webput);
	note: briefy used -tmpbackup on chgsed in PreHtmFixup (but not in PostHtmFixup);  then went to --nobackup on both;
	TITLE+ALT fixups not previously done on NOCI-site;  but can be, provided ALT-removal done only on lines having TITLE;  (these notes originally in webput)
	   TITLE+ALT:  only orchidshowINVITE08.htm had both TITLE and ALT (identical) -- have now removed ALT;
	   ALT-only:   eg: <input type=image src=ButtonMagenta-AddToCart.gif alt="Add to Cart" align=middle>	<--for Donate|Subscribe|AddToCart|ViewCart  ==FIX in ER==
	   ALT-only:   eg: <img src=NativeOrchidNews.gif alt="Native Orchid News:">
	   ALT-only:   eg: <img src=CHDonateNow4BA.gif border=0 alt="Donate Now Through CanadaHelps.org" align=middle>
	   ALT-only:   eg: <img src=LOGO-NOCI_130x130.gif alt="NOCI LOGO">
	   ALT-only:   eg: <IMG SRC="wa-StatsGraph.png" ALT="Usage summary for www.nativeorchid.org" HEIGHT=256 WIDTH=512>
	   TITLE-only: many cases;
	the TITLE+ALT-fixups devised in 2005May, were then in /er/webput, originally ALT seemed the norm so they were TITLE-adding;
	   on 2006-05-09: changed from only ALT= to only TITLE= in my local copy of HTML files (so Firefox shows them);  see /er/chg-alt-to-title;
	   on 2008-03-10, improved the ALT-removing cmd by restricting it to lines having TITLE;
	   on 2008-11-01, improved the ALT-adding   cmd by restricting it to lines lacking ALT  (eg: needed after an aborted webput);
	   on 2008-11-02, moved those fixups into WEBPUT-LL (as PreHtmFixup and PostHtmFixup),  and simplified the comments, hence these notes from the old comments;
(3)  Always use With-Deletion,  ie: eliminate having separate code for WITH/WITHOUT-DELETION:
	use:  -e (with-deletion)  so obsolete remote-files get cleaned up;
	use:  -n (only-newer)     to be safer wrt webalizer subfiles being updated;
	use:  -r (no-recursion)   since: (A) the absence isn't working in any event; (B) safer wrt webalizer subfiles;
(4)  Doing redirects via .htaccess:
	converted {debwendon,er,noci} .htaccess:  adding "##==ADD 301-REDIRECTS HERE==" at end,  then the recent redirects converted from $1-redirects/*
	(debwendon has 3 from 2008-06;  er has none;  noci has several from 2008-06 and 2008-10, plus menu* from 2008-09 done manually);
	SEE NOTES in webmv;

2008-11-03:  Testing after rewrite, on Debwendon:
	(1) Applying /debwendon/website-queue produced "Unknown command NOTE:" -- due to semicolon;  have fixed, I hope;
	(2) mysterious errormsg on mirror-cmds -- another problem-char in lftp-comment, COMMA or EQUALS??
		get "mirror: invalid option -- ,"  on comment of:  ##upload w/o delete  ==2008-10-26: back to -r, -n, and -e (with-deletion)
	(3) reconnect-woes on the "ls" cmd??
	note: the TITLE+ALT-fixups make NO changes upon /debwendon/website/*htm files, so aborting is pain-free;
	aborted, tried again with revised-comment-on-mirror-cmds, now getting reconnect-woes on first Upload (cgi-bin subdir), altho not on webalizer-Download;
	ControlPanel changes:
		renamed index.htm under ereimer.net;  Debwendon's index-htm-rename had been done, before errormsg;
		removed spurious .htaccess under remote-root, one level too high;
		removed .htaccess~ under ereimer.net & nativeorchid.org;
	REMOVING  "set ftp:list-options -a"  <--cured reconnect-woes--!!--
		it was supposed to solve hidden-dot-file problem, eg: .htaccess .htaccess~
		Apparently some conflict between that "set" and the "set ftp:list-empty-ok yes" to cure reconnect-woes??  ==Combine into one set-cmd??
	Fixed the rename of $1-ls  to  $1-ls-before,  to handle rerun-after-abort,  by making it conditional upon existence;
	NOTE: have finally uploaded revised cgi-bin/to-email-EKO.pl,  so retested feedback-form,  altho no need since only added a comment to invalidE routine:-)

2008-11-03:  Testing after rewrite, on NOCI all is well;  on Debwendon all is well;
2008-11-03:  on ER:
	upload was painfully slow in a new way;  first ANON-pixsearch.txt took nearly forever,  then later sitemap.xml took even longer, stuck at 33%, then at 47%...
	HostExcellence appears to be extremely sluggish (or was it Shaw?);  finally finished after approx 3-hours!!
	but all looks to be well.

2009-03-22:  had a puzzling result when I temporarily disabled uploading of several ER NOPERCART-demo dirs;
	the explanation:  webmv-Renames to IMG files were done (queued ftp cmds),  but HTML files were not uploaded,  ergo things were broken;
	BEWARE when wanting to upload excluding certain dirs with unfinished changes==!!==

2009-08-31: skipping of subdirs now done using OLDER/NEWER, which has workaround for faulty timestamp-updating on parent dir;  was using -ot;
	Note: my old comment on that line  "mv+rm update dir's mtime, -ot uses mtime"  suggests I did some testing on how mtime works on dir??
	but I wrote that on 2008-11-01, and haven't made any significant software upgrades since then, or have I...

2009-12-08:  because webalize-auto, which is run from WEBGETLOGS-ALL (cron-job), wants to upload just the wa-stats sub-dir and NOTHING BUT,
	modified to accept optional 2nd-param naming one subdir;  when specified, only that subdir is uploaded, and no ls is done  (even if other dirs have changes);
	re-arranged code, combining the loop-thru-subdirs with the do-rootdir, in order to achieve this;
	==could now safely skip the upload of wa-stats sub-dir when doing a "normal" (no 2nd-param) WEBPUT-LL, for slightly faster webput-upload;

2009-12-15:  PreHtmFixup, PostHtmFixup:
	added href=.*jpg --> href=\1htm fixups -- to cater to CGI-generated per-img-pages -- only for /er/website at least for now;
	==possibly time to scrap the ALT/TITLE fixups which catered only to netscape-4, seeing as how my nav-bar code rules out ns4;
	  could then use ALT as intended, for a meaningful name that non-image-enabled browser will show;  and only TITLE is used for my rollover-captions;
	  however since Google (images-bot) and accessibility-guidelines recommend that every IMG-tag have meaningful ALT, best to keep those fixups??
	  considered adding an ALT derived from filename, but one derived from TITLE is clearly more meaningful!!
2009-12-27:  skip DOWNLOADSTATS step when called from webput;  only do it on the nightly call from webalize-auto;
	==could also skip Uploading of wa-stats dir, except when it's the 2nd-param;

2010-03-14:  using Apache for local-testing, means needing PreFixup & PostFixup for .htaccess;
	(A) by chgsed-revising DOM<-->PROJ,  eg: ereimer.net<-->er;
	(B) by copying/renaming:  Pre: .htaccess-->.htaccess-LOCAL .htaccess-HOSTED-->.htaccess;  Post: .htaccess-LOCAL-->.htaccess;
	plan-(A) would be nicer if Apache-versions were compatible, but with v2.0 locally and v1.3 at HE there may be too many things to be revised??
	==webmv needs changes either way;  with (A) its redirects use PROJ;  with (B) it adds slightly different redirects to .htaccess AND .htaccess-HOSTED (messy);
	--DECIDED TO GO WITH plan-(A);  it turns out to be perfectly feasible even with v2.0 locally;  and may yet downgrade to v1.3 for even better HE-compatability??
	tested on er's .htaccess by:  http://ereimer.net-->http://er  ereimer\.net-->er$  AND nothing more was needed!!  ==no need for the v2 form of Include-spec!!
	--for cgi-bin, ScriptAlias in .htaccess is NOT the answer (gets "ScriptAlias not allowed here");  the following in httpd.conf works:
		Options +ExecCGI
		SetHandler cgi-script	<--AddHandler for by-suffix;  SetHandler for ALL files within that dir to be treated as CGI programs==!!==
		--THAT WORKS, but pixsearch needed revision to avoid redirecting to ereimer.net, due to its <BASE href=...
		--NOTE: now that I'm using Apache for local-testing, a leading-slash becomes the answer -- no longer any need for ../ sorts of things==!!==
	==the following belong in .htaccess, but then I've got 4 separate such files (at least plan-A means it isn't 8);  (will try to make first parts mostly identical)
	--Redirectmatch: tried it again at HE and it now works!!  SEE notes in noci's .htaccess, and the HE-ticket;
2010-03:
	discarded the ER-only PreHtmFixup:
	    chgsed -k -n -q '/href=http:/!s|\(href=[^>]*[0-9][0-9][0-9]\).jpg|\1.htm|g' "$@"	##revise href=.*3-digits.jpg to samename.htm
	    ONE-TIME REVISION:  cd /er/website;  chgsed -k --tmp '/href=http:/!s|\(href=[^>]*[0-9][0-9][0-9]\).jpg|\1.htm|g'  *htm */index.htm
	    it revised  pixCaption.htm  pixOrchids.htm  {20020523...20100309,AllenReimer,AuntHelen,BernieKroemer,BettySiren,KPLReimer,Sid}/index.htm;
	    note: 20060813/index-ALT.htm  KPLReimer/index-ALT.htm -- are OK, both consist entirely of <img src=...jpg> lines that need no revision;
	discarded the ER-only PostHtmFixup:
	    chgsed -k -n -q '/href=http:/!s|\(href=[^>]*[0-9][0-9][0-9]\).htm|\1.jpg|g' "$@"	##revise href=.*3-digits.htm to samename.jpg;  but not uppercase HREF==!!
	PreHtaFixup, PostHtaFixup: new routines to do the .htaccess revisions described above;  rely on new vars set by WEBVARS;
	    ONE-TIME REVERSABILITY-CHECK:
		for PROJ in debwendon er noci;do set /$PROJ/website; . /etc/sbin/WEBVARS; cd $1; cp .htaccess tmp;
		HOST=${URL#http://};        HOSTA=${HOST//./\\\\.}
		LOCHOST=${LOCURL#http://};  LOCHOSTA="${LOCHOST}\\\$"
		echo "==CWD:$(pwd); HOSTA=$HOSTA; LOCHOSTA=$LOCHOSTA;";
		chgsed "s|$URL|$LOCURL|g;s|$HOSTA|$LOCHOSTA|g" tmp;  dif .htaccess tmp |m;
		chgsed "s|$LOCURL|$URL|g;s|$LOCHOSTA|$HOSTA|g" tmp;  dif .htaccess tmp;
		done
		--all is well, after fixing the "escaping" of backslash and dollar-sign in HOSTA and LOCHOSTA;
	    ONE-TIME REVISION:
		for PROJ in debwendon er noci;do set /$PROJ/website; . /etc/sbin/WEBVARS; cd $1;
		HOST=${URL#http://};        HOSTA=${HOST//./\\\\.}
		LOCHOST=${LOCURL#http://};  LOCHOSTA="${LOCHOST}\\\$"
		chgsed -k --tmp "s|$URL|$LOCURL|g;s|$HOSTA|$LOCHOSTA|g" .htaccess;
		done
2010-03:
	==more notes that belong in .htaccess:
	Considered: switching from SSI-Include to PHP-Include, because same method throughout has appeal, PHP needed for FloraFind.Org;
	TO SWITCH TO PHP-INCLUDES:
		include-line from:	<!--#include virtual="/nav1-head.txt" -->
		include-line to:	<?php include($_SERVER["DOCUMENT_ROOT"] . "/nav1-head.txt"); ?>
		include-line ALT:	<?php include("./nav1-head.txt"); ?>					<==not as fool-proof when subdirs involved??
		.htaccess from:	Options +Includes;  AddHandler server-parsed  .htm
		.htaccess to:	AddType    application/x-httpd-php  .htm	<==Godawfully BAD advice!!
		.htaccess to:	AddHandler application/x-httpd-php  .htm	<==OLD advice, should work in apache-v1 or v2;  w/o messing-up text/html mimetype;
		.htaccess to:   AddHandler php5-script              .htm	<==NEW advice, but only for apache-v2;
		SEE ALSO:  /etc/httpd/conf.d/php.conf				<--to configure PHP for local-Apache
		HOSTEXCELLENCE: only explains a Control-Panel method:  see http://www.hostexcellence.com/index.php/v2/pages.manual7
		works on noci with local-Apache;
	==DECIDED TO STAY WITH:  .htm parsed for SSI-Includes;  .php for PHP-Directives==!!==
	==HOWEVER I MAY PREFER:  using .htm filenames for PHP-files on a Drupal website (florafind.org);  see notes in 404report==??==
2010-12-15:
	added special-case for the photo-upload dir;  BEWARE: lftp-mirror is broken wrt -e (deletion) option!!  need Download first before the Upload!!
	lftp-mirror seems Doubly-Broken;  now the Download clobbers the cgi-script, however this case is solved by -n option, so is dangerous rather than broken;
	make backup under /tmp/photo-upload;  also added -n option to the mirror-cmd;  -n appears to fix the problem??
2010-12-21:
	BEWARE: another nasty surprise: the DOWNLOAD step for photo-upload dir: inexplicably clobbered photo-upload.cgi file by transferring from hoster->local;
	had just used HE-ControlPanel to tidy-up removing photo-upload.cgi~ and that just might be relevant...
	no harm done, as they were identical;