#!/bin/bash ## JSNDMAG2WEB ######################################## # Sends mag image data from the WCOSS # to RZDM Site A and Site B (Boulder) # # Paula Freeman 5/2015 - New - substantially redesigned # from old "go" file version # Paula Freeman 6/2015 - move transfer files to "transfer" # subdir of status dir # Paula Freeman 3/2016 - expand section that reads values from # MAG_sync_table.tbl to include those removed from # exsendmag2web.sh.ecf, and pass in those fields # to exsendmag2web.sh.ecf in argument list. # Paula Freeman 9/2016 - If MODEL not found in the table, # exit in failure. # Paula Freeman 12/016 - Process oldest transfer files first # Anu Simon 04/2020 The jlog file reference is only commented # Left it there for future referenceee # Anu Simon 11/2021 The Name change to the script # Anu Simon 11/2021 Changes to conform to wcoss2 standards # Anu Simon 11/2022 Writing status files to dated directories - reversed back # 05/2024 SPA chnage to remove "export transfer_list" ####################################################### date export PS4='+ $SECONDS + ' set -x ####################################################### # The following variables could be defined in the # submission script (the ecf script), if # not they will take the default values which is set # for the NCO running enviroment ####################################################### export job=${job:-jmag.${MODEL:?}} export rzdm_envir=${rzdm_envir:-${envir:?}} export ncorzdm_username=${ncorzdm_username:-nwprod} # define variables for rzdm boulder export mag_web_dir=${mag_web_dir:-nco_mag} export site_b_envir=${site_b_envir:-${rzdm_envir}} export site_b_username=${site_b_username:-${ncorzdm_username}} export site_b_mag_web_dir=${site_b_mag_web_dir:-${mag_web_dir}} ## ####################################################### # Create temporary working directory ####################################################### export DATA=${DATAROOT:?}/${jobid:?} mkdir -p $DATA cd $DATA ##################################################### # Cycle is needed for the setPDY to make date # values available in the working directory # $cyc not used in MAG, default to previous hour ##################################################### export cyc=${cyc:-`date -u +%H -d "1 hour ago"`} export cycle=t${cyc}z ############################################################## # Set up the UTILITIES # Add path to utility scripts. # Run setpdy and initialize PDY variables ############################################################### setpdy.sh . ./PDY #################################### # Specify NET and RUN Name #################################### export NET=mag magName=`echo ${NET}_ver` NET_ver=${!magName} ############################################## # Define COM directories ############################################## export COMIN=${COMIN:-$(compath.py ${NET}/${NET_ver})/status} export COMOUT=${COMOUT:-$(compath.py ${NET}/${NET_ver})/gifs} #################################### # Specify Execution Areas #################################### export HOMEmag=${HOMEmag:-${PACKAGEROOT:?}/mag.${mag_ver:?}} export EXECmag=${EXECmag:-$HOMEmag/exec} export USHmag=${USHmag:-$HOMEmag/ush} export SCRIPTSmag=${SCRIPTSmag:-$HOMEmag/scripts} export FIXmag=${FIXmag:-$HOMEmag/fix} export PARMmag=${PARMmag:-$HOMEmag/parm} # enable extended pattern matching shopt -s extglob ########################################################## # $MODEL - specific model mnumonic corresponding to the table first # field # $TABLE - the table of data to use from $TABLEDIR # $TRANSFER_DIR - Directory containing transfer files to process #################################### export TABLEDIR=$PARMmag export TABLE=MAG_sync_table.tbl export model=$(echo $MODEL | tr '[:upper:]' '[:lower:]') if [ "${TARGET_SITE^^}" == 'A' ]; then export TRANSFER_DIR=${COMIN}/transfer_A elif [ "${TARGET_SITE^^}" == 'B' ]; then export TRANSFER_DIR=${COMIN}/transfer_B else err_exit "Invalid TARGET_SITE specified" fi mkdir -p $TRANSFER_DIR export DEBUG=0 ## log_it ########################################################### # Provides status and logging messages # Possible logging levels are "info warn debug err" # info & warn: echo messages provided # debug: echoes a message only if debug is turned on # err: invokes the standard error handler err_exit ########### log_it() { SEVERITY=$1 MESSAGE=$2 ZDATE="date -u" case "$SEVERITY" in info|warn) echo "[`${ZDATE}`] $MESSAGE" ;; debug) if [ "$DEBUG" -eq 1 ]; then echo "Debug:[`${ZDATE}`] $MESSAGE" fi ;; err) echo "[`${ZDATE}`] $MESSAGE" err_exit "$MESSAGE" ;; esac } ############################################################### ##Get Sync Params # Read the parameters required to set up the rsync from the table # # The table has entries separated by a vertical bar (|). The # first field in each entry is the model mnemonic and is # compared with the first argument of the command line to this # subroutine. # fields in the table are: # 1) model value # 2) Source Root directory # 3) Source Site directory # 4) Destination Host # 5) Destination User # 6) Destination Website Directory # 7) Site B Destination Host # 8) Site B Destination User # 9) Site B Destination Website Directory ################################################################ get_sync_params() { MODEL=$1 TABLE_FILE=$2 TARGET_SITE=$3 found_model=0 # Set to 1 if MODEL is found in TABLE_FILE exec 3< $TABLE_FILE # Set file descriptor 3 for the input file while read entry <&3 ;do # Take input from file assigned to file descriptor 3 # # Skip empty lines and comments # if [ "$entry" = "" ]; then # Empty input line continue fi skip=`echo $entry | awk 'index($1,"#") == 1 { print 1 } \ index($1,"#") != 1 { print 0 }'` if [ $skip -eq 1 ]; then # Comment line, process next input line continue fi log_it debug "process_request: Processing $entry" mname=`echo $entry | awk -F'|' 'index($1,"#") != 1 {print $1}'` log_it debug "process_request: Processing ${MNEMONIC} records in table" if [ $mname = ${MODEL} ]; then # Matching record. Extract fields. If required files are missing, error exit. found_model=1 log_it debug "process_request: record ${entry} matches model type" export gifdir=`echo $entry | awk -F'|' '{print $2}'` export model_dirname=`echo $entry | awk -F'|' '{print $3}'` export transfer_file_modelpat=`echo $entry | awk -F'|' '{print $10}'` transfer_file_modelpat=`echo $transfer_file_modelpat | sed -e "s/%/|/"` export srcWebsiteRoot=`eval echo $(echo $entry | awk -F'|' 'index($1,"#") != 1 {print $2}')` export srcWebsiteDir=`echo $entry | awk -F'|' 'index($1,"#") != 1 {print $3}'` export useTransFileList=`echo $entry | awk -F'|' 'index($1,"#") != 1 {print $11}'` if [ "${TARGET_SITE^^}" == 'A' ]; then echo "Sync to primary site" # These entries may have embedded environment variables, so evaluate it. export destUser=`eval echo $(echo $entry | awk -F'|' 'index($1,"#") != 1 {print $5}')` export destWebsiteDir=`eval echo $(echo $entry | awk -F'|' 'index($1,"#") != 1 {print $8}')` export destHost=`echo $entry | awk -F'|' 'index($1,"#") != 1 {print $4}'` elif [ "${TARGET_SITE^^}" == 'B' ]; then echo "Sync to site B" # These entries may have embedded environment variables, so evaluate it. export destUser=`eval echo $(echo $entry | awk -F'|' 'index($1,"#") != 1 {print $7}')` export destWebsiteDir=`eval echo $(echo $entry | awk -F'|' 'index($1,"#") != 1 {print $9}')` export destHost=`echo $entry | awk -F'|' 'index($1,"#") != 1 {print $6}'` fi if [ -z "${srcWebsiteRoot}" ]; then log_it err "No source website root directory specified." fi # srcWebsiteRoot is blank if [ -z "${srcWebsiteDir}" ]; then log_it err "No source website directory specified." fi # srcWebsiteDir is blank if [ "${destHost}" != "" ]; then if [ "${destUser}" != "" ]; then export REMOTE="${destUser}@${destHost}" else log_it err "No destination user specified." fi # dest user is blank else log_it err "No destination host specified." fi # destHost is blank if [ "${destWebsiteDir}" == "" ]; then log_it err "No destination directory specified." fi # destWebsiteDir is blank if [ "${srcWebsiteDir}" == "" ]; then log_it err "No source directory specified." fi log_it debug "================================" log_it debug " model: $MODEL" log_it debug " srcWebsiteRoot: $srcWebsiteRoot" log_it debug " srcWebsiteDir: $srcWebsiteDir" log_it debug " REMOTE: $REMOTE" log_it debug "================================" break fi #if the model mnemonic supplied in the command line matches the table entry done # while read # If $MODEL was not found in $TABLE_FILE, exit in failure if [ $found_model -eq 0 ]; then log_it err "FAIL: Model $MODEL not found in table." fi } #=============================================================================== # Keep synching as long as there are transfer files. # Check transfer files in status directory every 6 seconds for up to 6 minutes. #=============================================================================== env # Get the sync parameters get_sync_params $MODEL ${TABLEDIR}/$TABLE $TARGET_SITE # Look for a transfer list file: _yyyymmddcc_.transfer transfer_file_pattern=${transfer_file_modelpat}'_??????????_*.transfer' transfer_list=`ls -1rt ${TRANSFER_DIR}/${transfer_file_pattern} `; # A counter of 6-second intervals counter=0 # counter max is 60 x 6-seconds, which equals six minutes counter_max=60 # Looping every 6 seconds to look for a transfer_list for up to 6 minutes while : do if [[ -n $transfer_list ]]; then # Process all the transfer files in the transfer_list for transfer_file in $transfer_list; do export filename=`basename $transfer_file` yyyymmdd=`echo $filename | cut -d_ -f2 | cut -c-8` cycle=`echo $filename | cut -d_ -f2 |cut -c9-10` echo "==========================================" echo "Model: $MODEL cycle: $yyyymmdd $cycle" echo "==========================================" $SCRIPTSmag/exsendmag2web.sh $REMOTE $srcWebsiteRoot $srcWebsiteDir $destWebsiteDir $transfer_file $yyyymmdd $cycle err=$? if [[ $err == 0 ]]; then rm $transfer_file else echo "ERROR transferring GIF files listed in $transfer_file " fi done fi # Look for another transfer list after 6 seconds ((counter++)) echo "Try number $counter to look for transfer files..." sleep 6 transfer_list=$(ls -1rt ${TRANSFER_DIR}/${transfer_file_pattern}) # If transfer_list is NOT empty, which means new transfer files found, # then reset counter and immediately process newly found transfer files. # otherwise, do nothing. if [ -n "$transfer_list" ] then counter=0 else # Check counter to see 6-minute limit is reached. if [ $counter -eq $counter_max ] then echo "NOTHING TO PROCESS - $TRANSFER_DIR/${transfer_file_pattern} DOES NOT EXIST" break fi fi done ############################## # Remove the Temporary working directory ############################## if [ "${KEEPDATA^^}" != 'YES' ]; then rm -rf ${DATA:?} fi date