#!/bin/bash

export LANG=
export LC_ALL=

DEBUG=""
DRY=""
background=true
per_host_logfile=false
SSHDCMD="./sshgo.pl"
HOSTLIST=hostlist.inc

# for sshgo.pl: do not tunnel X Sessions
export SSHOPTIONS="-x"
# following does not work with older ssh version (woody)
# -o SetupTimeout=30 -o ConnectTimeout=20"

set_cmd() { :; }
cmd='date -u "+%F %T $HOSTNAME" ; uptime'
commandline_args="$*"

usage()
{
cat <<___ | less -e
$1
USAGE: $0 [options] [ -c "cmd" | -a "alias" ][pattern ...]
  -i            interactive (no backgrounding)
  -c "cmd"      command to be executed on all selected hosts
  -a "alias"    run one of the predefined aliases on all hosts
                "alias" is one of:
                
                keydist|k       distribute the sshdist keyfiles
                svnup|su        run "svn up" in lbcommon
                svnversion|sv   run "svnversion" in lbcommon
                symlinkd|sd     run the symlink dryrun script in lbcommon
                symlink|sy      run the symlink script in lbcommon
                symlinko|so     run the symlink-other script in lbcommon
                svnupsymd|sud   run svn up & symlink dryrun in lbcommon
                ntpq            run ntpq -n -c peers
                aptup|au        run apt-get update
                sys2wiki|s2w    run sys2wiki script. run with --per-host-log.
                                after this run ./sys2wiki.sh

  -L "hostlist" use a different hostlist (default: ./hostlist.inc)

  --per-host-log
                generate tmp.<hostname>.out files for each host
                [ 'make clean' to get rid of them again ]

  --default     on all hosts do: $cmd

  --dry         print sshgo routing info
                shows but does not execute remote commands

  --debug       adds -v to sshgo options
                prefix remote command with set -vx

  --stdin "somefile"
                for each ssh command, redirect stdin: <somefile
                (default: /dev/null)
                uploading one file becomes:
		$0 -c "cat > outfile" --stdin infile
		or
		$0 -c "tar -C /some/where xzf -" --stdin tmp.tgz
  --upload "somefile"
                short for $0 -c "cat > somefile" --stdin somefile
		this overrides a previously given -c "cmd",
		and is overriden by a later given -c "cmd".
  --umask "umask"
                prepends command with 'umask "umask";'

Typically, the given command will be executed on all hosts in parallel,
reducing the effect of netwok latency.
You can interrupt this script (and all backgrounded ssh sessions)
with ctrl-C.

pattern is a shell pattern filter applied to the target system ids
which are read from the hostlist.
Format is like:
  "lb_*.root" for all linbit hosts,
  "sd_*"      for all solve hosts etc...
___
[[ $1 ]] && exit 1 || exit 0
}

[[ $1 ]] || set -- help

default_pattern="*.root"
stdin=/dev/null
umask=""
while [[ $1 ]]; do
	case $1 in
	help|-h|-help|--help)
		usage
		;;
	--default) :;; # use default command
	-i)     background=false
		stdin=""
		;;
	-L)     HOSTLIST=$2; shift;;
	--per[_-]host[_-]log)
		per_host_logfile=true;;
	--umask)
		umask "$2" || exit
		umask=$2
		shift
		;;
	--upload|--stdin)
		if test -r "$2"; then
			stdin="$2"
			[[ $1 == --upload ]] && cmd=$(printf "cat > %q" "$2")
			shift
		else
			echo >&2 "$2 not readable"
			exit 1
		fi
		;;
	--dry)     DRY=x ;;
        --debug)   DEBUG=x ;;
	-c)
		cmd=$2
		[[ -z $cmd ]] && background=false
		shift
		;;
	-a)
		shift
		case $1 in
		keydist|k)
			default_pattern="*"
			set_cmd() {
				key=$( printf "%q" "$(< AUTHKEYS/$id)" )
				cmd="umask 077 && echo $key > .ssh/authorized_keys2.new && chmod 600 .ssh/authorized_keys2.new && mv -v .ssh/authorized_keys2.new .ssh/authorized_keys2"
			}
			;;
		svnup|su)
			set_cmd() { :; }
			cmd="[ -d /usr/local/lbcommon -a -x /usr/bin/svn ] && { cd /usr/local/lbcommon ; svn up ; } || true"
			;;
		svnversion|sv)
			set_cmd() { :; }
			cmd="[ -d /usr/local/lbcommon -a -x /usr/bin/svnversion ] && { cd /usr/local/lbcommon ; svnversion . ; } || true"
			;;
		svnupsymd|sud)
			set_cmd() { :; }
			cmd="[ -d /usr/local/lbcommon -a -x /usr/bin/svn ] && { cd /usr/local/lbcommon ; svn up ; ./symlink-etc.sh -d ; } || true"
			;;
		symlinkd|sd)
			set_cmd() { :; }
			cmd="[ -d /usr/local/lbcommon ] && { cd /usr/local/lbcommon ; ./symlink-etc.sh -d ; } || true"
			;;
		symlink|sy)
			set_cmd() { :; }
			cmd="[ -d /usr/local/lbcommon ] && { cd /usr/local/lbcommon ; ./symlink-etc.sh ; } || true"
			;;
		symlinko|so)
			set_cmd() { :; }
			cmd="[ -d /usr/local/lbcommon ] && { cd /usr/local/lbcommon ; ./symlink-other.sh ; } || true"
			;;
		ntpq)
			set_cmd() { :; }
			cmd="ntpq -n -c peers"
			;;
		aptup|au)
			set_cmd() { :; }
			cmd="apt-get update"
			;;
		sys2wiki|s2w)
			set_cmd() { :; }
			cmd="[ -x /usr/local/lbcommon/tools/sys2wiki2.pl ]  && /usr/local/lbcommon/tools/sys2wiki2.pl || true"
			;;
		*)
			usage "unknown alias $1"
			;;
		esac
		;;
	-*)
		usage "unknown option: $1"
		;;
	*) break;;
	esac
shift
done

pattern="${1:-$default_pattern}"
[[ $pattern == *.* ]] || pattern=$pattern.root

run_on()
{
	ip="$1"
	id="$2"
	host="$3"
	fhost=$( printf "%-15s " $host )

	$background && trap 'echo "$host got HUP signal" ; exit' HUP
	
	set_cmd
	cmd="${DEBUG:+set -xv;}${umask:+umask "$umask";}$cmd"
	
	if $background ; then
		if $per_host_logfile ; then
			$SSHDCMD ${DRY:+ -d} ${DEBUG:+ -v} -T -t $ip $id "$cmd" <"$stdin" 2>&1 |
			tee tmp.$host.out | cat -n | sed -u -e "s/^/$fhost/"
		else
			$SSHDCMD ${DRY:+ -d} ${DEBUG:+ -v} -T -t $ip $id "$cmd" <"$stdin" 2>&1 |
			cat -n | sed -u -e "s/^/$fhost/"
		fi
	else
		printf >&2 "connecting to %-15s (%s, %s)\n" "$host" "$id" "$ip"
		if [[ -z $stdin ]] ; then
			# typically we won't redirect stdin for non-backgrounded sessions,
			# since we probably want to be interactive then.
			$SSHDCMD ${DRY:+ -d} ${DEBUG:+ -v} -t $ip $id "$cmd" 2>&1
		else
			$SSHDCMD ${DRY:+ -d} ${DEBUG:+ -v} -t $ip $id "$cmd" <"$stdin" 2>&1
		fi
	fi

	test ${PIPESTATUS[0]} = 0 && status="[OK]" || status="[FAIL]"
	printf >&2 "%-15s %-6s (%s, %s)\n" "$host" "$status" "$id" "$ip"
}

set -m # I need job control to be able to conveniently forward the ctrl-C
trap 'echo >&2 signal caught, user interrupted' HUP INT QUIT PIPE TERM
date >&2 -u "+# started at %F %T: $commandline_args"$'\n#---'

while read -u3 ip id host
do
	case "$id" in
	$pattern)
		if $background; then
			( run_on $ip $id $host ) &
		else
			run_on $ip $id $host
		fi
		;;
	esac
done 3< $HOSTLIST

if ! wait; then
	# ps xf
	for pid in $( jobs -p ); do
		kill -HUP -$pid &>/dev/null
	done
	# ps xf
fi

date >&2 -u "+# finished at %F %T"