#!/bin/sh
# Standard job for transferring directories between systems of the same type
# on the NOAA/NWS/NCEP Weather and Climate Operational Supercomputer Systems.
# The following environment variables will be used by the job if set:
# 
#   HOMEtransfers, DATA, COMROOT, RUN, SENDCOM, SENDECF, KEEPDATA,
# 
#   RSYNC_DIRECTION*:   Whether to PUSH data out or PULL it in (default PUSH)
#   RSYNC_MAX_SECONDS:  Maximum IO timout of each rsync connection (default 300)
#   RSYNC_MAX_ATTEMPTS: Number of times to attempt each rsync (default 3)
#   RSYNC_OPTIONS:      A string of additional rsync command-line parameters
#   RSYNC_DEST_ROOT:    If defined, will be prepended to all destination
#                       directory paths. This is mainly used for testing.
#   TRANSFER_LIST:      A file containing a list of directories and optional
#                       include/exclude patterns.
#                       If the PACKAGE* ecFlow variable is set, TRANSFER_LIST is
#                           ${HOME%PACKAGE%}/parm/transfer_${RUN}.list
#                       Otherwise, it defaults to
#                           ${HOMEtransfers}/parm/transfer_${RUN}.list
# 
# The "P" progress meter is not entirely reliable since it is based on the number
# of files rsync is currently aware of.  Because rsync scans directories in an
# incrementally recursive fashion, this is not necessarily the total number
# of files that need to be sync'd.  For instance, it may discover some large files
# in a deeply nested directory later on which may skew the progress calculation.
# 
#   * RSYNC_DIRECTION, PACKAGE, and CPUS can be set as ecFlow variables

date
export PS4=' $SECONDS + '
set -x

export DATA=${DATA:-${DATAROOT:?}/${jobid:?}}
mkdir -p $DATA
cd $DATA

export jlogfile=${jlogfile:-${DATA}/jlogfile.${jobid}}

export cycle=${cycle:-t${cyc}z}
setpdy.sh 10 1
. ./PDY

export MONCUR=${PDY:0:6}
export MONPREV=$(date -d "$PDY -1 month" +%Y%m)
export MONNEXT=$(date -d "$PDY +1 month -3 day" +%Y%m)
export YEARCUR=${PDY:0:4}
export YEARPREV=$(date -d "$PDY -1 year" +%Y)
export YEARNEXT=$(date -d "$PDY +16 day" +%Y)

export SENDCOM=${SENDCOM:-NO}
export SENDECF=${SENDECF:-YES}
export SENDWEB=${SENDWEB:-YES}

export PARMtransfers=${HOMEtransfers:?}/parm
export FIXtransfers=${HOMEtransfers:?}/fix
export USHtransfers=${HOMEtransfers:?}/ush

# Establish rsync timeout and retry limits
export RSYNC_MAX_SECONDS=${RSYNC_MAX_SECONDS:-300}
export RSYNC_MAX_ATTEMPTS=${RSYNC_MAX_ATTEMPTS:-3}

# Determine which direction to move the data (PUSH or PULL)
export RSYNC_DIRECTION=${RSYNC_DIRECTION:-PUSH}

# Determine which directories/files to transfer
export TRANSFER_LIST=${TRANSFER_LIST:-$PARMtransfers/transfer_${RUN:?}.list}
if [ ! -f $TRANSFER_LIST ]; then
   echo "$TRANSFER_LIST not exist, check the transfer subdirectory!"
   dir=`dirname $TRANSFER_LIST`
   fileName=`basename $TRANSFER_LIST`
   TRANSFER_LIST=${dir}/transfer/${fileName}
fi

ecfID=$(echo ${PBS_JOBID} | awk -F"." '{print $1}')
export OUTPUTFILE=${PBS_OUTPUTFILE:-${PBS_O_WORKDIR}/${PBS_JOBNAME}.o${ecfID}}
export stdout_file=$(readlink -m $OUTPUTFILE)

# JY export blacklist_file=/usrx/local/sys/blacklist/service.prod
export blacklist_file=${HOME}/blacklist/service.prod

export webserver="nwprod@ncointra"

if [ "$envir" = "prod" ]; then
  export sqlite_database=${sqlite_database:-"/home/nco/www/htdocs/idsb/transfers_wcoss/transfers_wcoss_status.db"}
else
  export sqlite_database=${sqlite_database:-"/home/nco/www/htdocs/idsb/transfers_wcoss_test/transfers_wcoss_status.db"}
fi

#############################################################
# execute the script

if [ "$RUN" = "ecf_scripts" ]; then
  err_exit "This job is not set up to check ecf_scripts"
else
  echo "Cleaning up the $sqlite_database ..."
  echo "saving $sqlite_database to ${sqlite_database}.bkup.$PDY"
  ssh ${webserver:?} "mv $sqlite_database ${sqlite_database}.bkup.$PDY"
  ssh ${webserver:?} "cp ${sqlite_database}.clean ${sqlite_database}"
fi
export err=$?; err_chk
#############################################################

postmsg "$jlogfile" "$0 completed normally"

########################################
# Remove the Temporary working directory
########################################
if [[ $KEEPDATA != "YES" ]]; then
  rm -rf $DATA
fi

date