#!/bin/bash
#
# JABS - Just Another Backup Script
# Copyright 2007-2010 Toixland.org
# Based on VMware vcbSnapAll script
#
# Last edited: 25 oct 2010
#
JABS_VERSION="1.2j"

#####################################################################
#
# SUMMARY
#
# This script will export virtual machines known to a particular
# host into a specified target directory from where they can
# be picked up by backup software.
# Virtual machine selection will be performed in two phases:
# 1. select VMs based on a search criteria (see vcbSnap -r option)
# 2. filter VMs using a list of good VMs (session VMs list)
# After the selection, the virtual machines are checked against
# some condition (i.e. no snapshots already exists) and then
# backup will be performed using vcbMounter utility.
# Summary log and detailed logs for every virtual machines will
# be written and optionally emailed by SMTP.
# 
# To use this utility you need a configuration file that sets
# the needed session environment (session configuration file).
# See the jabs.conf.dist file for a list of configuration params.
#
# You need to set-up VCB environment too, editing the file
#    /etc/vmware/backuptools.conf
#

#####################################################################
#
# USAGE
#
#    jabs -s <session_file> [-u <username>][-p <password>][-h <url>]
#                           [-r <destination>][-a <select>]
#                           [-L <verbosity>]
#                           [-d <debug_flag>] [-t <test_flag>]
#         <session_file> file containing session configuration
#         <username>     username to connect to the host
#                        (default: USERNAME)
#         <password>     password to connect to the host
#                        (default: PASSWORD)
#         <url>          <hostname>[:<port>]
#                        (default: VCHOST)
#         <destination>  backup destination
#                        (see vcbMounter -r option)
#         <select>       search specifier to identify VMs groups
#                        (default: all powered on VMs)
#                        (see vcbVmName -s option)
#         <verbosity>    verbosity used by vcbMounter command [0-6]
#                        (default: 3)
#         <debug_flag>   Must be set to '1' if you want to enable
#                        debug mode (default: 0)
#         <test_flag>    Must be set to '1' if you want to enable
#                        test mode (default: 0)
#
#    All parameters, except the <session_file>, are defined into the
#    <session_file> too, which is the only required parameter.
#    Some default values (USERNAME, PASSWORD and VCHOST) are defined
#    in the VCB environment (see /etc/vmware/backuptools.conf).
#    If multiple values will be found for a parameter, the rule is:
#    - Command line parameters override session file values
#    - Session file paramaters override default values
# 


#####################################################################
#
# THIS IS THE BACKUP SCRIPT!
#
# YOU DON'T NEED TO CHANGE ANY OF THE FOLLOWING LINES.
#
# You must change the values into the session configuration files,
# not here!
#
#####################################################################


#####################################################################
#
# VCB library intialization
#
vcb_libs=/usr/lib/vmware/vcb
cfg=${VCB_CONFIG_FILE:-"/etc/vmware/backuptools.conf"}
. "$vcb_libs/plugins/lib"
if [ -e "$cfg" ] ; then
    . "$cfg"
fi

#####################################################################
#
# Global variables and constants
#

#
# Exit codes used by this script
#
ERR_OK=0
ERR_NOTALLVMSBACKEDUP=1
ERR_GETVMLIST=2
ERR_NOVMSFOUND=3
ERR_NOSESSIONVMSFOUND=4
ERR_SESSIONDESTROOT=5
ERR_USAGE=6
ERR_SIGNALTRAPPED=7
ERR_NOFREESPACE=8

#
# Global arrays to hold the list of MoRefs, DisplayNames, IPaddress
# UUIDs and VMX paths for VMs to be backed up.
# Set by create_vm_list, read by backup_vms.
#
declare -a SELECTED_VMMOREFS
declare -a SELECTED_VMNAMES
declare -a SELECTED_VMIPADDRS
declare -a SELECTED_VMUUIDS
declare -a SELECTED_VMXFILES

#
# Global array to hold the list of VMs for the session
#
declare -a SESSION_VMLIST

#
# Global array to hold the list of VMs found on the host
#
declare -a FOUND_VMLIST

#
# Some usefull and default global vars
#
VMSFOUND=0
tmp=${TEMPDIR:-"/tmp"}
SESSION_LOG_DIRECTORY=/var/log/vmware/jabs-$$
SESSION_QUERY="powerstate:on"
SESSION_HOST="$VCHOST"
SESSION_DESTINATION_FREE=1
SESSION_ABORT_IF_SNAPSHOT=1
SESSION_TEST_MODE=0
SESSION_CLEARDEST_TRY=3
SESSION_CLEARDEST_SLEEP=5m
SUMMARYLOG="/dev/null"

#
# For debug purpose only
#
DEBUG_MODE=0

#
# Commands used in this script
# Better if you don't change them
#
AWK="/bin/awk"
CAT="/bin/cat"
COMM="/usr/bin/comm"
ECHO="/bin/echo"
GREP="/bin/grep"
MKDIR="/bin/mkdir"
MV="/bin/mv"
DF="/bin/df"
RM="/bin/rm"
SLEEP="/bin/sleep"
SORT="/bin/sort"
SYNC="/bin/sync"
TAIL="/usr/bin/tail"
TOUCH="/bin/touch"
VCBMOUNTER="/usr/sbin/vcbMounter"
VCBVMNAME="/usr/sbin/vcbVmName"
VMWARECMD="/usr/bin/vmware-cmd"
VMKFSTOOLS="/usr/sbin/vmkfstools"
WC="/usr/bin/wc"


#####################################################################
#
# FUNCTION: init
#
# Description:
# Sanity check and initialization.
#
# Arguments: 
# None. Checks global variables.
# 
# Result:
# Returns ERR_OK on success or ERR_USAGE on failure.
#
function init
{
    local rv=$ERR_OK
    local msg=""

    #
    # Setting the required parameters
    #
    if [ -n "${INTERCEPT_d}" ] ; then 
	DEBUG_MODE="${INTERCEPT_d}"
    fi
    if [ -n "${INTERCEPT_s}" ] ; then 
	SESSION_PREFS="${INTERCEPT_s}"
    fi
    #
    # OK, lets go!
    #
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "[DEBUG] ENTERING init FUNCTION"
    fi
    #
    # First of all we want to check the required session parameters
    #
    if [ -z "$SESSION_PREFS" ] ; then
	msg="Session file is unset"
    elif ! [ -s "$SESSION_PREFS" ] ; then
	msg="Session file is invalid"
    else
	#
	# Load the session specific variables
	#
        . "$SESSION_PREFS"
	#
	# Set the optional command line arguments
	# so that they will override the session file
	#
	if [ -n "${INTERCEPT_a}" ] ; then 
	    SESSION_QUERY="${INTERCEPT_a}"
	fi
	if [ -n "${INTERCEPT_h}" ] ; then 
	    SESSION_HOST="${INTERCEPT_h}"
	fi
	if [ -n "${INTERCEPT_p}" ] ; then 
	    PASSWORD="${INTERCEPT_p}"
	fi
	if [ -n "${INTERCEPT_r}" ] ; then 
	    SESSION_DESTINATION_ROOT="${INTERCEPT_r}"
	fi
	if [ -n "${INTERCEPT_u}" ] ; then 
	    USERNAME="${INTERCEPT_u}"
	fi
	if [ -n "${INTERCEPT_d}" ] ; then 
	    DEBUG_MODE="${INTERCEPT_d}"
	fi
	if [ -n "${INTERCEPT_t}" ] ; then 
	    SESSION_TEST_MODE="${INTERCEPT_t}"
	fi
	#
	# Let we take a little formal check over them
	#
	if [ -z "$SESSION_NAME" ] ; then
	   msg="Session file is invalid: session name unset"
	elif [ -z "$SESSION_VMLIST" ] ; then
	   msg="Session file is invalid: VMs list is empty"
	#
	# Username
	#
	elif [ -z "$USERNAME" ] ; then
	    msg="User name is unset"
	#
	# Password
	#
	elif [ -z "$PASSWORD" ] ; then
	    msg="User password is unset"
	#
	# Destination directory
	#
	elif [ -z "$SESSION_DESTINATION_ROOT" ] ; then
	    msg="Destination directory is unset"
	elif ! [ -d "$SESSION_DESTINATION_ROOT/" ] ; then
	    msg="Destination directory is invalid"
	#
	# Destination directory free space
	#
	elif [ -z "$SESSION_DESTINATION_FREE" ] ; then
	    msg="Destination free space is unset"
	#
	# Query
	# Defaulted to "powerstate:on"
	#
	elif [ -z "$SESSION_QUERY" ] ; then
	    msg="Search specifier is unset"
	#
	# Hostname
	# Defaulted to $VCHOST
	#
	elif [ -z "$SESSION_HOST" ] ; then
	    msg="Host name is unset"
	fi
    fi
    #
    # Free space checks
    # Defaulted to 1 GB
    #
    if [ -z "$msg" ] ; then
	if [ $SESSION_DESTINATION_FREE -gt 0 ] ; then
            declare destination_fs=$($DF "$SESSION_DESTINATION_ROOT" | $TAIL -n1 | $AWK '{ print $1 }')
	    if [ "$destination_fs" = "-" ] ; then
		if [ $DEBUG_MODE -eq 1 ] ; then
		    $ECHO "[DEBUG] --- VMFS destination file system found"
		fi
		declare -i destination_free=$($VMKFSTOOLS -P "$SESSION_DESTINATION_ROOT" | $GREP Capacity | $AWK ' BEGIN { FS = "," } { print $2 }' | $AWK ' { print $1 }')
		let destination_free/=1024
		let destination_free/=1024
		let destination_free/=1024
	    else
		if [ $DEBUG_MODE -eq 1 ] ; then
		    $ECHO "[DEBUG] ---  Standard destination file system found"
		fi
		declare -i destination_free=$($DF -P -B 1G "$SESSION_DESTINATION_ROOT" | $TAIL -n1 | $AWK '{ print $4 }')
	    fi
	    if [ $DEBUG_MODE -eq 1 ] ; then
		$ECHO "[DEBUG] --- Free space on destination file system: $destination_free GB"
	    fi
	    if [ $SESSION_DESTINATION_FREE -gt $destination_free ]  ; then
		msg="Too few space in destination directory"
	    fi
	fi
    fi
    #
    # Initialization finished.
    # Lets see if there was errors.
    #
    if [ -z "$msg" ] ; then 
	if [ -e "$SESSION_LOG_DIRECTORY" ] ; then
	   $RM -Rf "$SESSION_LOG_DIRECTORY"
	fi
	$MKDIR "$SESSION_LOG_DIRECTORY"
        SUMMARYLOG="$SESSION_LOG_DIRECTORY"/"session.log"
	rv=$ERR_OK
    else
	if [ "$msg" = "Too few space in destination directory" ] ; then
	   rv=$ERR_NOFREESPACE
	else
	   $ECHO "Missing or invalid arguments: ${msg}."
           $ECHO "Usage: jabs -s <session_file> [-u <username>][-p <password>][-h <url>]"
	   $ECHO "                              [-r <destination>][-a <select>]"
	   $ECHO "                              [-L <verbosity>][-d <flag>][-t <flag>]"
           $ECHO "    <session>   file containing session specific configuration"
           $ECHO "    <username>  needed to connect to the host"
           $ECHO "    <password>  needed to connect to the host"
           $ECHO "    <verbosity> [0-6](default: 0)"
           $ECHO "    <flag>      set to '1' to enable debug or test mode (default: 0)"
           $ECHO "    Session override parameters:"
           $ECHO "       <url>          <hostname>[:<port>] (default: $HOSTNAME)"
           $ECHO "       <destination>  path to destination directory"
           $ECHO "       <select>       (see vcbSnap -a option)(default: powered on)"
	   $ECHO
	   rv=$ERR_USAGE
	fi
	if ! [ -z "$SESSION_EMAIL_SERVER" ] ; then
	   msg="$msg"
	   $SENDEMAIL -q -f "$SESSION_EMAIL_FROM" \
		-t "$SESSION_EMAIL_TO" \
		-u "Initialization error" \
		-m "$msg" \
		-s "$SESSION_EMAIL_SERVER"
	fi
    fi
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "[DEBUG] Line argument parsing returns:"
	$ECHO "[DEBUG]    --> INTERCEPT_d = ${INTERCEPT_d}"
	$ECHO "[DEBUG]    --> INTERCEPT_s = ${INTERCEPT_s}"
	$ECHO "[DEBUG]    --> INTERCEPT_a = ${INTERCEPT_a}"
	$ECHO "[DEBUG]    --> INTERCEPT_h = ${INTERCEPT_h}"
	$ECHO "[DEBUG]    --> INTERCEPT_p = ${INTERCEPT_p}"
	$ECHO "[DEBUG]    --> INTERCEPT_r = ${INTERCEPT_r}"
	$ECHO "[DEBUG]    --> INTERCEPT_u = ${INTERCEPT_u}"
	$ECHO "[DEBUG]    --> INTERCEPT_t = ${INTERCEPT_t}"
	$ECHO "[DEBUG] Session variables setting:"
	$ECHO "[DEBUG]    --> SESSION_PREFS=$SESSION_PREFS"
	$ECHO "[DEBUG]    --> USERNAME=$USERNAME"
	$ECHO "[DEBUG]    --> PASSWORD=$PASSWORD"
	$ECHO "[DEBUG]    --> SESSION_HOST=$SESSION_HOST"
	$ECHO "[DEBUG]    --> SESSION_NAME=$SESSION_NAME"
	$ECHO "[DEBUG]    --> SESSION_QUERY=$SESSION_QUERY"
	$ECHO "[DEBUG]    --> SESSION_VMLIST=$SESSION_VMLIST"
	$ECHO "[DEBUG]    --> SESSION_DESTINATION_ROOT=$SESSION_DESTINATION_ROOT"
	$ECHO "[DEBUG]    --> SESSION_DESTINATION_FREE=$SESSION_DESTINATION_FREE"
	$ECHO "[DEBUG] --- Debug mode active, forcing test mode on"
	SESSION_TEST_MODE=1
	$ECHO "[DEBUG] EXITING init FUNCTION"
    fi
    return $rv
}



#####################################################################
# 
# FUNCTION: create_vm_list
#
# Description:
# Fill a list of all virtual machines matching a particular
# search criteria.
#
# Arguments
# $1: Virtual machine search specifier
#
# Result:
# Returns ERR_OK on success, ERR_GETVMLIST or ERR_NOVMSFOUND on
# failure.
#
# Sets SELECTED_VMMOREFS, SELECTED_VMNAMES, SELECTED_UUIDS,
# SELECTED_IPADDRS and FOUND_VMLIST arrays on success.
# Also set the VMSFOUND global variable.
#
function create_vm_list 
{
    local searchspec="$1"
    local sessionVmFile="${tmp}"/jabs-ses-$$
    local selectedVmFile="${tmp}"/jabs-sel-$$
    local vmxFileList="${tmp}"/jabs-vmxlist
    local tmpFile="${tmp}"/jabs-tmp-$$
    local line
    local total
    local rv
    local prefix
    local value
    local counter

    #
    # First of all I'll go to create a temp file with
    # the name of the powered on VMs hosted here
    # The list goes into $selectedVmFile.
    #
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "[DEBUG] ENTERING create_vm_list FUNCTION"
	$ECHO "[DEBUG] --- $VCBVMNAME -h $SESSION_HOST -u $USERNAME -p $PASSWORD -s $searchspec"
    fi
    $VCBVMNAME -h "$SESSION_HOST" -u "$USERNAME" -p "$PASSWORD" \
              -s "$searchspec" > "$tmpFile"
    rv=$?    
    if [ $DEBUG_MODE -eq 1 ] ; then
        $ECHO "[DEBUG] --- Return value: $rv"
        $ECHO "[DEBUG] --- $CAT $tmpFile"
        $ECHO "----------------------------"
	$CAT "$tmpFile"
        $ECHO "----------------------------"
    fi
    if [ "$rv" != "0" ] ; then
	$RM -f "$tmpFile"
	return $ERR_GETVMLIST
    fi
    if [ -e "$selectedVmFile" ] ; then
	$RM -f "$selectedVmFile"
    fi
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO -n "[DEBUG] --- Begin file analysis"
    fi
    while read line
    do
        prefix="${line%%:*}"
        value="${line#*:}"
        if [ "$prefix" = "name" ] ; then
	    let VMSFOUND+=1
            FOUND_VMLIST[${#FOUND_VMLIST[*]}]="$value"
            $ECHO "$value">>"$selectedVmFile"
	    if [ $DEBUG_MODE -eq 1 ] ; then
		$ECHO -n "."
	    fi
	fi
    done <"${tmpFile}"
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO ".DONE"
	$ECHO "[DEBUG] --- $VMSFOUND virtual machines found"
    fi
    if [ "$VMSFOUND" = 0 ] ; then
	$RM -f "$tmpFile"
	return $ERR_NOVMSFOUND
    fi
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "[DEBUG] --- $SORT -f -o $selectedVmFile <$selectedVmFile"
    fi
    $SORT -f -o "$selectedVmFile" <"$selectedVmFile"
    $ECHO "========================================================" >>"$SUMMARYLOG"
    $ECHO "Local VMs found matching the search criteria ["$searchspec"]" >>"$SUMMARYLOG"
    counter=0
    while read line
    do
	let counter+=1
        $ECHO "   ["$counter"] - ${line}" >>"$SUMMARYLOG"
    done <"${selectedVmFile}"
    $ECHO " " >>"$SUMMARYLOG"
    #
    # Now I'll build the list of session VMs into a file.
    # The list file will be $sessionVmFile.
    #
    if [ -e "$sessionVmFile" ] ; then
	$RM -f "$sessionVmFile"
    fi
    counter=0
    total=${#SESSION_VMLIST[*]}
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO -n "[DEBUG] --- Building session VMs list file $sessionVmFile"
    fi
    while [ $counter -lt $total ]
    do
#        $ECHO "${SESSION_VMLIST[$counter]}" >>"$sessionVmFile"
        $GREP "${SESSION_VMLIST[$counter]}" "${selectedVmFile}">>"$sessionVmFile"
        let counter+=1
	if [ $DEBUG_MODE -eq 1 ] ; then
	    $ECHO -n "."
	fi
    done
    if [ $DEBUG_MODE -eq 1 ] ; then
        $ECHO ".DONE"
    fi
    #
    # Then I'll see which VMs match the VMs listed into the
    # session file. The match goes into $selectedVmFile.
    #
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "[DEBUG] --- $SORT -f -o $sessionVmFile <$sessionVmFile"
    fi
    $SORT -f -o "$sessionVmFile" <"$sessionVmFile"
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "[DEBUG] --- $COMM -1 -2 $sessionVmFile $selectedVmFile >$tmpFile"
    fi
    $COMM -1 -2 "$sessionVmFile" "$selectedVmFile" >"$tmpFile"
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "[DEBUG] --- $MV -f $tmpFile $selectedVmFile"
    fi
    $MV -f "$tmpFile" "$selectedVmFile"
    if [ $DEBUG_MODE -eq 1 ] ; then
        $ECHO "[DEBUG] --- $CAT $selectedVmFile"
        $ECHO "----------------------------"
	$CAT "$selectedVmFile"
        $ECHO "----------------------------"
    fi
    #
    # For every line in $selectedVmFile fills the VM arrays
    #
    if [ $DEBUG_MODE -eq 1 ] ; then
        $ECHO "[DEBUG] --- Getting the virtual machines MOREFS, UUIDS and IPADDRS"
    fi
    while read line
    do
	name="${line}"
	SELECTED_VMNAMES[${#SELECTED_VMNAMES[*]}]="${line}"
	#
	# see if we got a moref or a uuid...
	#
	if [ $DEBUG_MODE -eq 1 ] ; then
	    $ECHO "[DEBUG] --- $VCBVMNAME -h $SESSION_HOST -u $USERNAME -p $PASSWORD -s name:${line} > $tmpFile"
	fi
        $VCBVMNAME -h "$SESSION_HOST" -u "$USERNAME" -p "$PASSWORD" \
                  -s "name:${line}" > "$tmpFile"
        rv=$?    
	if [ $DEBUG_MODE -eq 1 ] ; then
	    $ECHO "[DEBUG] --- Return value: $rv"
	fi
        if [ "$rv" != "0" ] ; then
            $RM -f "$tmpFile"
            return $ERR_GETVMLIST
        fi
        if [ $DEBUG_MODE -eq 1 ] ; then
            $ECHO "[DEBUG] --- $CAT $tmpFile"
            $ECHO "----------------------------"
            $CAT "$tmpFile"
            $ECHO "----------------------------"
            $ECHO -n "[DEBUG] --- Begin file analysis"
        fi
	while read line
	do
		prefix="${line%%:*}"
		value="${line#*:}"
		if [ "$prefix" = "moref" ] ; then
		    SELECTED_VMMOREFS[${#SELECTED_VMMOREFS[*]}]="$value"
		elif [ "$prefix" = "uuid" ] ; then 
		    SELECTED_VMUUIDS[${#SELECTED_VMUUIDS[*]}]="$value"
		elif [ "$prefix" = "ipaddr" ] ; then 
		    SELECTED_VMIPADDRS[${#SELECTED_VMIPADDRS[*]}]="$value"
		fi
        	if [ $DEBUG_MODE -eq 1 ] ; then
		    $ECHO -n "."
		fi
	done <"${tmpFile}"
        if [ $DEBUG_MODE -eq 1 ] ; then
	    $ECHO ".DONE"
        fi
        $RM -f "$tmpFile"
    done <"${selectedVmFile}"
    #
    # Clean up the disk space
    #
    if [ -e "$selectedVmFile" ] ; then
	$RM -f "$selectedVmFile"
    fi
    if [ -e "$sessionVmFile" ] ; then
	$RM -f "$sessionVmFile"
    fi
    if [ -e "$tmpFile" ] ; then
	$RM -f "$tmpFile"
    fi
    #
    # prepare the temporary file to store the .VMX file list
    #
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "[DEBUG] --- Prepare the .VMX list file"
	$ECHO "[DEBUG] --- $VMWARECMD -l >${vmxFileList}"
    fi
    $VMWARECMD -l >"${vmxFileList}"
    if [ $DEBUG_MODE -eq 1 ] ; then
        $ECHO "[DEBUG] --- $CAT $vmxFileList"
        $ECHO "----------------------------"
        $CAT "$vmxFileList"
        $ECHO "----------------------------"
    fi
    #
    # Get the VMX file path for every VMs to be backed up
    # and print selected VMs infos into the summary log
    #
    counter=0
    total=${#SELECTED_VMMOREFS[*]}
    if [ "$total" = 0 ] ; then
	$RM -f "$tmpFile"
	return $ERR_NOSESSIONVMSFOUND
    fi
    $ECHO "Local VMs selected for backup:" >>"$SUMMARYLOG"
    if [ $DEBUG_MODE -eq 1 ] ; then
        $ECHO "[DEBUG] --- $total VMs selected for backup"
        $ECHO -n "[DEBUG] --- Getting .VMX path for selected VMs"
    fi
    while [ $counter -lt $total ]
    do
	#
	# Search the display name of the VM into the .VMX files
	#
	while read line
	do
	   if [ $DEBUG_MODE -eq 1 ] ; then
	       $ECHO -n "."
	   fi
	   if ! [ -z "`grep displayName ${line} | grep ${SELECTED_VMNAMES[$counter]}`" ] ; then
		SELECTED_VMXFILES[$counter]="${line}"
		break
	   fi
	done <"${vmxFileList}"
	if [ $DEBUG_MODE -eq 1 ] ; then
	    $ECHO -n "*"
	fi
        $ECHO "   ["`expr $counter + 1`"] - "${SELECTED_VMNAMES[$counter]}":"${SELECTED_VMIPADDRS[$counter]}":"${SELECTED_VMMOREFS[$counter]}":"${SELECTED_VMUUIDS[$counter]} >>"$SUMMARYLOG"
        $ECHO "         "${SELECTED_VMXFILES[$counter]} >>"$SUMMARYLOG"
        let counter+=1
    done
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO ".DONE"
	$ECHO "[DEBUG] EXITING create_vm_list FUNCTION"
    fi
    $ECHO " " >>"$SUMMARYLOG"
    return $ERR_OK
}



#####################################################################
#
# FUNCTION: get_vmsubdir_name
#
# Description:
# Get a directory name for a VM specified by its Display Name.
# Also, replace "special characters" in the Display Name with underscores.
# If SESSION_DESTINATION_UNIQUE is greater than zero then the directory name
# will be generated appending a counter to it.
#
# Writes the directory name to stdout.
#
# Arguments:
# $1: Display Name of the VM (used as a basis for the new dir. name)
#
# Result:
# None.
#
function get_vmsubdir_name
{
    local name="$1"
    local newdir
    local count
    declare -i count=1

    #
    # Use the virtual machine's display name as a basis.
    # Cut out any characters from the display name that can cause
    # problems with subsequent backup scripts.
    #
    newdir=`$ECHO $name | tr [:blank:]\"\'\\/\* _`
    #
    if [ $SESSION_DESTINATION_UNIQUE -gt 0 ] ; then
	#
	# Now check if the subdir name is unique
	# If not, just make it unique by appending a serial number
	#
	if [ -e "${SESSION_DESTINATION_ROOT}/${newdir}" ] ; then
	    while [ -e "${SESSION_DESTINATION_ROOT}/${newdir}-$count" ]
	    do
		count=count+1
	    done
	    newdir="${newdir}-$count"
	fi
    fi
    $ECHO $newdir
}



#####################################################################
#
# FUNCTION: backup_vms
#
# Description:
# Read a list of MoRefs from stdout and back up all the VMs in the list.
#
# Arguments:
# None.
#
# Result:
# Returns ERR_OK or ERR_NOTALLVMSBACKEDUP if backup failed for at least
# one virtual machine.
#
function backup_vms
{
    local total
    local ok
    local failed
    local starttime=`date`
    local rv=$ERR_OK
    local i
    local t
    declare -i total=0 ok=0 failed=0 i=0 t=0
    
    if [ $DEBUG_MODE -eq 1 ] ; then
        $ECHO "[DEBUG] ENTERING backup_vms FUNCTION"
    fi
    #
    # Adjust some spurious parameters
    #
    #if [ -n "$DATASTORE" ] ; then
    #	DS_ARG="-C \"$DATASTORE\""
    #else
    #	DS_ARG=""
    #fi
    #UNPARSED_ARGUMENTS=`$ECHO "$UNPARSED_ARGUMENTS" |$AWK -F'--' '{ print $1 }'`
    DS_ARG=""
    UNPARSED_ARGUMENTS="-L $SESSION_LOG_LEVEL -Q 0"
    #
    # Count the number of VMs we are dealing with
    #
    total=${#SELECTED_VMMOREFS[*]}
    $ECHO "Backing up ${total} matching VMs." >>"$SUMMARYLOG"
    $ECHO "Per VM log files will be in ${SESSION_LOG_DIRECTORY}."
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO -n "[DEBUG] --- Begin backup of VMs"
    fi
    while [ $i -lt $total ]
    do
	if [ $DEBUG_MODE -eq 1 ] ; then
            $ECHO -n "."
	fi
	$ECHO -n `date`": Exporting VM "${SELECTED_VMNAMES[$i]}"..." >>"$SUMMARYLOG"
	VMSUBDIR=`get_vmsubdir_name "${SELECTED_VMNAMES[$i]}"`
	detaillog="$SESSION_LOG_DIRECTORY"/"$VMSUBDIR"
	$RM -f "${detaillog}-*"
	$TOUCH "${detaillog}-running"
	if [ $rv != $ERR_SIGNALTRAPPED ] ; then
		if [ "`check_vm_pre $i ${detaillog}-running`" -eq 1 ] ; then
		    $ECHO "[BACKUP]" $VCBMOUNTER -h "$SESSION_HOST" -u "$USERNAME" -p "*****" \
				     -a moref:"${SELECTED_VMMOREFS[$i]}" \
				     -r "$SESSION_DESTINATION_ROOT"/"$VMSUBDIR" $DS_ARG \
       		   	             $UNPARSED_ARGUMENTS >> "${detaillog}-running" 2>&1
		    if [ $SESSION_TEST_MODE -eq 0 ] ; then
			#
			# Prepare the destination directory
			#
			t=0
			while ( [[ $t -lt $SESSION_CLEARDEST_TRY ]] && [[ -e "${SESSION_DESTINATION_ROOT}/${VMSUBDIR}" ]] )
			do
			    if [ $t -gt 0 ] ; then
				$ECHO "failed" >>"${detaillog}-running"
			    fi
			    t=t+1
			    $ECHO -n "[BACKUP] Removing destination directory ($t)..." >>"${detaillog}-running"
			    $RM -Rf "${SESSION_DESTINATION_ROOT}/${VMSUBDIR}"
			    $SYNC
			    $SLEEP $SESSION_CLEARDEST_SLEEP
			done
			if [ -e "${SESSION_DESTINATION_ROOT}/${VMSUBDIR}" ] ; then
			    $ECHO "failed" >>"${detaillog}-running"
			    $ECHO "[BACKUP] Destination directory still exists, backup may abort" >>"${detaillog}-running"
			else
			    if [ $t -gt 0 ] ; then
				$ECHO "success" >>"${detaillog}-running"
			    fi
			fi
			#
			# FINALLY WE CAN LAUNCH THE BACKUP COMMAND!!
			#
			trap "i=$total; rv=$ERR_SIGNALTRAPPED" INT TERM EXIT
			eval $VCBMOUNTER -h '$SESSION_HOST' -u '$USERNAME' -p '$PASSWORD' \
					-a moref:'${SELECTED_VMMOREFS[$i]}' \
					-r '$SESSION_DESTINATION_ROOT'/'$VMSUBDIR' $DS_ARG \
					$UNPARSED_ARGUMENTS >> "${detaillog}-running" 2>&1
			trap - INT TERM EXIT
			if [ $rv != $ERR_SIGNALTRAPPED ] ; then
			    rv=$?
			fi
		    else
			rv=$ERR_OK
		    fi
		else
		    rv=$ERR_NOTALLVMSBACKEDUP
		fi
	fi
	#
	# Check the result
	#
	if [ $rv != $ERR_OK ] ; then
	    if [ $rv != $ERR_SIGNALTRAPPED ] ; then
		$MV "${detaillog}"-running "${detaillog}"-failed
		$ECHO FAILED >>"$SUMMARYLOG"
		rv=$ERR_NOTALLVMSBACKEDUP
		failed=failed+1	    
	    else
		$MV "${detaillog}"-running "${detaillog}"-aborted
		$ECHO ABORTED >>"$SUMMARYLOG"
	    fi
	else
	    # Post backup checks
	    if [ "`check_vm_post $i ${detaillog}-running`" -eq 1 ] ; then
		$MV "${detaillog}"-running "${detaillog}"-ok
		$ECHO SUCCEEDED >>"$SUMMARYLOG"
		ok=ok+1
	    else
		$MV "${detaillog}"-running "${detaillog}"-failed
		$ECHO FAILED >>"$SUMMARYLOG"
		failed=failed+1
	    fi
	fi
	#
	# Set the right return value
	#
	if [ $failed -gt 0 ] ; then
	    if [ $rv != $ERR_SIGNALTRAPPED ] ; then
		rv=$ERR_NOTALLVMSBACKEDUP
	    fi
	fi
	i=i+1
    done
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO ".DONE"
    fi
    #
    # Output the summary log
    #
    $ECHO "========================================================" >>"$SUMMARYLOG"
    $ECHO "Session name     : $SESSION_NAME" >>"$SUMMARYLOG"
    $ECHO "Destination dir  : $SESSION_DESTINATION_ROOT" >>"$SUMMARYLOG"
    $ECHO "Query done       : $SESSION_QUERY" >>"$SUMMARYLOG"
    $ECHO "Backup start time: $starttime" >>"$SUMMARYLOG"
    $ECHO "Backup end time  :" `date` >>"$SUMMARYLOG"
    $ECHO " " >>"$SUMMARYLOG"
    $ECHO "           Total number of VMs found: $VMSFOUND" >>"$SUMMARYLOG"
    $ECHO "       Total number of VMs processed: $total" >>"$SUMMARYLOG"
    $ECHO "Number of VMs backed up successfully: $ok" >>"$SUMMARYLOG"
    $ECHO "                Number of VMS failed: $failed" >>"$SUMMARYLOG"
    $ECHO " " >>"$SUMMARYLOG"
    $ECHO "For per-VM logs, see directory ${SESSION_LOG_DIRECTORY}." >>"$SUMMARYLOG"
    $ECHO "========================================================" >>"$SUMMARYLOG"
    $ECHO " " >>"$SUMMARYLOG"
    if [ $DEBUG_MODE -eq 1 ] ; then
	$ECHO "----------------------------"
	eval $CAT '$SESSION_LOG_DIRECTORY/*-*'
	$ECHO "----------------------------"
	$CAT "$SUMMARYLOG"
        $ECHO "[DEBUG] EXITING backup_vms FUNCTION"
    fi
    return $rv
}



#####################################################################
#
# FUNCTION: check_vm_pre
#
# Description:
# Performs some checks on a virtual machine before the export.
#
# Arguments:
# $1: index order of the virtual machine referred to SELECED_* arrays
# $2: per-VM log file to be written
#
# Result:
# Returns 1 if then virtual machine passes all the tests, 0 otherwise.
#
function check_vm_pre
{
    local index=$1
    local logfile="$2"
    local retval=1
    local vmstate

    #
    # Check the state returned from vmware-cmd. It must be a valid state.
    #
    vmstate="`$VMWARECMD ${SELECTED_VMXFILES[$index]} getstate | $AWK -F' ' '{ print $3 }'`"
    case "$vmstate" in
	"on")
	    retval=1
	;;
	"off")
	    retval=1
	;;
	*)
	    $ECHO "[PRE BACKUP CHECK] Undefined state returned: $vmstate" >>"$logfile"
	    $ECHO "Skipping virtual machine" >>"$logfile"
	    retval=0
	;;  
    esac
    #
    # If state is good, lets do the other checks...
    #
    if [ $retval -eq 1 ] ; then
	#
	# Snapshot checking
	#
	if [ $SESSION_ABORT_IF_SNAPSHOT -eq 1 ] ; then
#	    if ! [ -z "`$VMWARECMD ${SELECTED_VMXFILES[$index]} hassnapshot | $AWK -F' ' '{ print $3 }'`" ] ; then
	    if ! [ "`$VMWARECMD ${SELECTED_VMXFILES[$index]} hassnapshot | $AWK -F' ' '{ print $3 }'`" -eq 0 ] ; then
		$ECHO "[PRE BACKUP CHECK] Virtual machine snapshot present and SESSION_ABORT_IF_SNAPSHOT" >>"$logfile"
		$ECHO "Skipping virtual machine" >>"$logfile"
		retval=0
	    fi
	fi
    fi
    #
    # Output the result
    #
    $ECHO $retval
}

#####################################################################
#
# FUNCTION: check_vm_post
#
# Description:
# Performs some checks on a virtual machine after the export.
#
# Arguments:
# $1: index order of the virtual machine referred to SELECED_* arrays
# $2: per-VM log file to be written
#
# Result:
# Returns 1 if then virtual machine passes all the tests, 0 otherwise.
#
function check_vm_post
{
    local index=$1
    local logfile="$2"
    local retval=1
    local vmstate

    #
    # Check the state returned from vmware-cmd. It must be a valid state.
    #
    vmstate="`$VMWARECMD ${SELECTED_VMXFILES[$index]} getstate | $AWK -F' ' '{ print $3 }'`"
    case "$vmstate" in
	"on")
	    retval=1
	;;
	"off")
	    retval=1
	;;
	*)
	    $ECHO "[POST BACKUP CHECK] Undefined state returned: $vmstate" >>"$logfile"
	    retval=0
	;;  
    esac
    #
    # If state is good, lets do the other checks...
    #
    if [ $retval -eq 1 ] ; then
	#
	# Snapshot checking
	#
	if [ $SESSION_ABORT_IF_SNAPSHOT -eq 1 ] ; then
#	    if ! [ -z "`$VMWARECMD ${SELECTED_VMXFILES[$index]} hassnapshot | $AWK -F' ' '{ print $3 }'`" ] ; then
	    if ! [ "`$VMWARECMD ${SELECTED_VMXFILES[$index]} hassnapshot | $AWK -F' ' '{ print $3 }'`" -eq 0 ] ; then
		$ECHO "[POST BACKUP CHECK] Virtual machine snapshot present and SESSION_ABORT_IF_SNAPSHOT" >>"$logfile"
		$ECHO "Snapshot must be manually removed" >>"$logfile"
		retval=0
	    fi
	fi
    fi
    #
    if [ $retval -eq 1 ] ; then
	#
	# I don't trust the vcbMounter exit code anymore!!
	# Let's examine the per-VM log in search of errors  
	#
	local magicword="error]"
	if [ "`$GREP $magicword $logfile | $WC -l`" -gt 0 ] ; then
                $ECHO "[POST BACKUP CHECK] Errors found in log file" >>"$logfile"
                retval=0
	fi
    fi
    #
    # Output the result
    #
    $ECHO $retval
}



#####################################################################
#
# FUNCTION: error_exit
#
# Description:
# Check for non-zero return status of a command. If a non-zero return
# status is encountered, clean up temp files, print an error message and
# exit with an appropriate error code.
#
# Arguments:
# $1: Return value to evaluate. Should be one of the ERR_ constants defined
#     in this script
#
# Result:
# None. Will terminate the shell script in case of an error.
#
function error_exit #<errval=$1>
{
    local retval="$1"
    local msg

    if [ "$retval" != "$ERR_OK" ] ; then
	case "$retval" in
	"$ERR_NOTALLVMSBACKEDUP")
	    msg="Backup did not succeed for all VMs"
	    ;;
	"$ERR_GETVMLIST")
	    msg="Could not retrieve a list of VMs to back up"
	    ;;
	"$ERR_NOVMSFOUND")
	    msg="No VMs the specified search criteria were found"
	    ;;
	"$ERR_NOSESSIONVMSFOUND")
	    msg="No local VMs found for this session"
	    ;;
	"$ERR_USAGE")
	    msg="Script was invoked with illegal or missing arguments"
	    ;;
	"$ERR_SIGNALTRAPPED")
	    msg="SIGNAL trapped, exiting"
	    ;;
	"$ERR_NOFREESPACE")
	    msg="Too few space in destination directory"
	    ;;
	*)
	    msg="Unspecified error during backup. Please refer to logs"
	    ;;
	esac
	$ECHO "${msg}." >&2
	$ECHO "${msg}."
	$ECHO
	$ECHO "EXITING..." >>"$SUMMARYLOG"
	#
	# Send log by email if there was a real error
	#
	if [ "$retval" != "$ERR_USAGE" ] ; then
	    msg="$msg"
	    send_mail "$msg"
	fi
	exit $retval
    fi
}



#####################################################################
#
# FUNCTION: send_mail
#
# Description:
# Send an email with the summary log
#
# Arguments:
# $1: subject text
#
# Result:
# None.
#
function send_mail
{
    local subject="$1"
    local body

    subject="$SESSION_NAME: $subject"
    body="Host: $SESSION_HOST"
    body="$body\nSession: $SESSION_NAME"
    if ! [ $SESSION_TEST_MODE -eq 0 ] ; then
       body="$body (TEST MODE)"
    fi
    body="$body\n"
    body="$body\nRefer to logs ($SESSION_LOG_DIRECTORY) for detailed information"
    if ! [ -z "$SESSION_EMAIL_SERVER" ] ; then
       $SENDEMAIL -q -f "$SESSION_EMAIL_FROM" \
           -t "$SESSION_EMAIL_TO" \
           -u "$subject" \
           -m "$body" \
           -s "$SESSION_EMAIL_SERVER" \
           -a "$SUMMARYLOG"
    fi
}



#####################################################################
#
# MAIN
#
$ECHO "JABS $JABS_VERSION - Just Another (fucking) backup script" 
#
# We intercept some parameters here.
#
intercept_arguments "a:h:p:r:s:u:d:t:" "$@"
#
# Variable checking and initialization
#
$ECHO "Initializing..."
init
rv=$?
error_exit $rv
$ECHO "JABS $JABS_VERSION - Just Another Backup Script" >"$SUMMARYLOG"
set -o nounset

#
# Virtual machines selection
#
$ECHO "Creating virtual machines list..."
create_vm_list "$SESSION_QUERY"
rv=$?
error_exit $rv

#
# Virtual machines backup
#
$ECHO "Backing up virtual machines..."
backup_vms
rv=$?
error_exit $rv

#
# Script completion
#
send_mail "Backup completed successfully"
$ECHO "BACKUP COMPLETED SUCCESSFULLY!"