#!/bin/sh -x
#
# 12/9/97 Changed to execute from /nfsuser/g06/.../tdlutil instead of
# /we21kh directory.  Also changed to remove state and awips headers.
#
# 7/22/98 Updated to handle the change from GOES9 to GOES10 for
# the WESTSCP data.
#
# 11/22/1999 Update for Y2K changes.  NESDIS began sending a
# 4-digit year in the filename.
#
# 3/21/2000 Converted to run on the IBM.
# 
# 5/10/2007 made easier to use by passing in arguments for 
# example ./scp_prep.sh EAST 2007 07 06 18 02  
# arguments:         domain year month prev-month day hour
#
# 8/10/2007 converted to run operationally for one hour.
#
# 9/10/2010 scpedit.f was recompiled. Script was retested.
#
# 9/24/2010 script edited and retested for operational processing.
#
# 12/2/2010 codes scpedit.f fmtscp.f and scppack.f edited and recompiled
#           to MOS-2000 standards. Also insertion of lmp_scp_dup.pl
#           to eliminate observations from duplicate bulletins.
#
# 01/6/2010 eliminate some redundant variables and warning messages.
#
#12/21/2011 modification for production include combining 
#           fmtscp.f, scpedit.f, and scppack.f into one fortran routine
#           which includes a call to the perl script lmp_scp_dup.pl within 
#           scpedit.f.  The new fortran code is to be called lmp_scpprep.f.
#           
#01/17/2012 JEG Modified touch command to fix bug, change executable names and
#               location to be exec instead of sorc.
#02/10/2012 MWS Modified script to accomodate various code walk-through changes.
#               The case statement of required inputs was removed because
#               all inputs can either be found or determined from variables in 
#03/09/2012 JEG Modified script to use headers files with name 
#               lmp_SCP_[east,west]_headers.lst instead of lmp_SCP_[east,west]_headers
#03/19/2012 JEG Modified script to save original tcus_[east,west] file with .${HOURm1}
#               on the end of the filename.
#03/29/2012 JEG Modified script to only provide one date to lmp_scpprep.  Previously
#               we supplied two dates, but they were the same.  The codes have been 
#               changed to expect only one date as input.  
#03/30/2012 JEG Modified script to clean up some comments based on code walkthrough
#               comments, removed "test2" from filename, etc.
#06/12/2012 DTM Added plus one hour to SCP_date and updated lmp_scp_retrieve.pl
#               accordingly.
#06/13/2012 JSI Added treatment for prepack.SCP.txt which has only header or 
#               (header and ZZZZ).   
#06/25/2012 JEG Modified to:
#               - add comment above renaming of prepack file
#               - change prints to echos for consistencies
#               - remove redirect of print/echo statements to $pgmouts.  These should
#               just be echos.  We redirect only executable output to $pgmout.   
#               - made warnings in the case of a problem with SCP to be consistent
#               and provide complete information. 
#               - commented out setting err and calling err_chk after lmp_scpprep
#               executable, and added comment explaining this similar to what is
#               done in exlmp_prep.sh.sms after calling lmp_radprep.            
#
# ====================================================================
#
#         PURPOSE: This script reads, edits, and packs one hour of SCP data.
#                  The resulting packed file is then used operationally for
#                  the gridded obs sky cover observation and for the LAMP
#                  sky cover SCP observation predictor.
#
# ====================================================================
#
set -x
echo LMPLOG: `date` - begin job lmp_scpprep.sh
#
#######################################################################
#                PROCESS STEP 0.   
#                NOTE: The date is GMT time.
#                GOESreg [FROM PREP] ='east' or 'west'
#                year='YYYY'
#                month='MM'
#                day='DD'
#                HOURm1 [FROM PREP] ='HH'
#######################################################################

year=`echo $PDYmh1 | cut -c 1-4`
month=`echo $PDYmh1 | cut -c 5-6`
day=`echo $PDYmh1 | cut -c 7-8`

# added 6-11-12 to account for the current day that goes with the current hour
currday=`echo $PDY | cut -c 7-8`

#######################################################################
#                1. Check domain "GOESreg"
#######################################################################

case $GOESreg in
east) echo "EAST SCP PROCESSING"
;;
west) echo "WEST SCP PROCESSING"
;;
esac

#######################################################################
#                2. Print the variable "indate" used to retrieve
#                   the raw archived SCP data.
#######################################################################

indate=`echo $PDYmh1`
echo "SCP PROCESSING FOR REG: $GOESreg DATE:$indate"

#######################################################################
#                3. Define monchar, premonchar, premon, postmonchar, 
#                   premnd, and mndy:
#######################################################################

case $month in
01) monchar=JAN; premonchar=DEC; premon=12; postmonchar=FEB; premndy=31; mndy=31
;;
02) monchar=FEB; premonchar=JAN; premon=01; postmonchar=MAR; premndy=31; mndy=28
;;
03) monchar=MAR; premonchar=FEB; premon=02; postmonchar=APR; premndy=28; mndy=31
;;
04) monchar=APR; premonchar=MAR; premon=03; postmonchar=MAY; premndy=31; mndy=30
;;
05) monchar=MAY; premonchar=APR; premon=04; postmonchar=JUN; premndy=30; mndy=31
;;
06) monchar=JUN; premonchar=MAY; premon=05; postmonchar=JUL; premndy=31; mndy=30
;;
07) monchar=JUL; premonchar=JUN; premon=06; postmonchar=AUG; premndy=30; mndy=31
;;
08) monchar=AUG; premonchar=JUL; premon=07; postmonchar=SEP; premndy=31; mndy=31
;;
09) monchar=SEP; premonchar=AUG; premon=08; postmonchar=OCT; premndy=31; mndy=30
;;
10) monchar=OCT; premonchar=SEP; premon=09; postmonchar=NOV; premndy=30; mndy=31
;;
11) monchar=NOV; premonchar=OCT; premon=10; postmonchar=DEC; premndy=31; mndy=30
;;
12) monchar=DEC; premonchar=NOV; premon=11; postmonchar=JAN; premndy=30; mndy=31
;;
*) echo "WARNING: Invalid number of month"
;;
esac

#######################################################################
#                4. Correct mndy and premndy (Feb/Mar) for leap year
#######################################################################

let dif=${year}%4                # if 2011 dif=3
let century=${year}%100          # if 2011 century=11
if [ $century == 0 ]; then       # if year is 1900,2000,2100 etc.
  let dif=${year}%400             # dif=0 for 2000,2400(premndy=29)
fi
if [ $dif == 0 ]; then           # 2000,04,08,12,16 etc.
  if [ $monchar == 'MAR' ]; then
    let $premndy=29
  fi
  if [ $monchar == 'FEB' ]; then
    let $mndy=29
  fi
fi

#######################################################################
#                5. Setup various GOES East and West 
#                   variables and copy the character string
#                   lmp_SCP_east_headers.lst or lmp_SCP_west_headers.lst
#                   from the /parm sub-directory.
#######################################################################

if [ $GOESreg == 'east' ]; then
  FILCOAST=e
  GOES=GOESE
  export FILCOAST GOES
elif [ $GOESreg == 'west' ]; then
  FILCOAST=w
  GOES=GOESW
  export FILCOAST GOES
fi

#######################################################################
#                Copy files containing headers
#######################################################################

cp $PARMlmp/lmp_SCP_${GOESreg}_headers.lst lmp_SCP_${GOESreg}_headers.lst

#######################################################################
#                6. Define the HOUR-DAY Boundary used to 
#                   detect and eliminate duplicate bulletins
#                   based on DATmh1 and DATmh2.
#                   ex. DATmh1 = 2011090523
#                   The values of HOURm1 and HOURM2 are already evaluated.
#######################################################################

export daym1hour=`echo $DATmh1 |cut -c7-8`
export daym2hour=`echo $DATmh2 |cut -c7-8`

pre40=40       # example: echo "280940" > date1
past20=20      # example: echo "281020" > date2

echo "$daym2hour$HOURm2$pre40" > date1
echo "$daym1hour$HOURm1$past20" > date2

echo "TO DETECT AND ELIMINATE DUPLICATE BULLETINS, THE OBSERVATION TIME"
echo "MUST RANGE FROM `cat date1` TO `cat date2`."

#######################################################################
#                PROCESS STEP 1.   
#                STEP 1A. READ IN THE RAW SCP DATA.
#                The SCP data from the most recent hour of file
#                /dcom/us007003/$indate/wtxtbul/tcus_east or west is retrieved
#                using the perl script scp_retrieve.pl and outputted to file
#                SCP_raw_out.txt. Extraneous characters are then removed from the file
#                before further processing.
#
#                NOTE: The file $indate/wtxtbul/tcus_east or west 
#                represents data for a 24 hour day in GMT time.
#
#                Construct the input list "SCP_date"
#######################################################################

echo "KNES" > SCP_date
# note: add a space in front of the two day/hour outputs to better
# match the date/hour from input lines
echo " $day$HOURm1" >> SCP_date
echo " $currday$HOUR" >> SCP_date   # output the current day and hour

cp /dcom/us007003/$indate/wtxtbul/tcus_${GOESreg} lmp_tcus_${GOESreg}.${HOURm1}

$USHlmp/lmp_scp_retrieve.pl SCP_date lmp_tcus_${GOESreg}.${HOURm1}

if [ $? -ne 0 ]; then
  echo "WARNING: Raw SCP data was not retrieved.  Zero-byte file created for tcus_${GOESreg} ${DATmh1}: EXITING SCP SCRIPT"
  touch lmp_sc${FILCOAST}_mdlp.${HOURm1}   # 0 BYTE OUTPUT FILE
  exit 1   ##  YES! exit from script when raw data not present
fi

## EDIT #1 Remove ^M from end of every line.
tr -d "\r" < SCP_raw_out.txt > UNF_GOES${GOESreg}SCP.${HOURm1}.edit1

## EDIT #2 Remove additional extraneous characters from every line
sed -e 's///g' UNF_GOES${GOESreg}SCP.${HOURm1}.edit1 > UNF_GOES${GOESreg}SCP.${HOURm1}.edit2


#######################################################################
#                STEP 1B. HEADER AND DUPLICATE CHECK FOR SCP FILE
#                Retrieve the header records to check if requested headers
#                are present. Also check for duplicates.
#                ### example:TCUS43 KNES 180222 ###
#######################################################################

fgrep -f lmp_SCP_${GOESreg}_headers.lst UNF_GOES${GOESreg}SCP.${HOURm1}.edit2 > SCP_retrieved_${GOESreg}_headers
check1_diff=0

##  check1 = number of bulletins
##  check1_unique = number of unique bulletins
##  check1_total_flag = 1 for the expected number of bulletins 

check1=`cat SCP_retrieved_${GOESreg}_headers | wc | cut -c7-8`
check1_unique=`sort SCP_retrieved_${GOESreg}_headers | cut -c1-6 | uniq | wc | cut -c7-8`
check1_total_flag=1

##        Check the number of headers [5 EAST HEADERS, 4 WEST HEADERS]
##        (DETECTION ONLY)

if [ $GOESreg == 'east' ]; then
   if [ $check1 -ne 5 ]; then
      echo "Warning: NUMBER OF READ SCP $GOESreg HEADERS IS NOT EQUAL TO 5; $check1 SCP HEADERS READ."
      check1_total_flag=0
   fi
elif [ $GOESreg == 'west' ]; then
   if [ $check1 -ne 4 ]; then
      echo "Warning: NUMBER OF READ SCP $GOESreg HEADERS IS NOT EQUAL TO 4; $check1 SCP HEADERS READ."
      check1_total_flag=0
   fi
fi

if [ $check1_total_flag -eq 1 ]; then
   echo "THE CORRECT NUMBER OF SCP $GOESreg HEADERS ($check1) WERE READ."
fi

##        Duplicate Header Check 
##        (DETECTION ONLY)

check1_diff=`expr $check1 - $check1_unique`
if [ $check1_diff -eq 0 ]; then
   echo "NO DUPLICATE HEADERS FOUND ON $GOESreg SCP FILE"
else
   echo "Warning: THERE ARE $check1_diff DUPLICATE SCP $GOESreg HEADERS FOUND ON $GOESreg SCP FILE"
   echo "         THE DUPLICATE BULLETINS NEED TO BE REMOVED FROM FILE $GOESreg SCP FILE"
fi

###    WHEN PROCESSING ONLY ONE HOUR THE INPUT day/hour IS
###    ALWAYS THE SAME AS THE HEADER DAY/HOUR

#######################################################################
#                PROCESS STEPS 2 - 4
#                The code lmp_scpprep.f runs subroutines ftmscp.f, 
#                scpedit.f and scppack.f.
#                1. The code fmtscp.f re-formats the SCP data from the 
#                   raw NESDIS ASCII product.
#                2. The code scpedit.f quality controls the SCP ASOS 
#                   cloud reports before being packed.
#                3. SCP Observations not matching a predetermined
#                   range of observation times are deleted using the
#                   perl script: lmp_scp_dup.pl. The Perl
#                   script is called within scpedit.f
#                4. The code scppack.f packs an ASCII file generated
#                   from scpedit.f to a TDLPACK file. 
#######################################################################

#######################################################################
#                RUN LMP_SCPPREP TO GET THE SCP DATA
#######################################################################

echo LMPLOG: `date` - start running lmp_scpprep.f

export pgm=lmp_scpprep
. prep_step
export XLFUNIT_30="UNF_GOES${GOESreg}SCP.${HOURm1}.edit2"  # INPUT FOR fmtscp.f
export XLFUNIT_40="FMTSCP_GOES${GOESreg}SCP.${HOURm1}"  # OUTPUT FROM fmtscp.f/INPUT FOR scpedit.f
export XLFUNIT_45="SCPEDIT_GOES${GOESreg}SCP.${HOURm1}"  # OUTPUT FROM scpedit.f/INPUT FOR lmp_scp_dup.pl
export XLFUNIT_50="prepack.SCP.txt"  # OUTPUT FROM lmp_scp_dup.pl/INPUT FOR scppack.f 
export XLFUNIT_55="lmp_sc${FILCOAST}_mdlp.$HOURm1" # OUTPUT FROM scppack.f

##  Input character string links requried to run lmp_scp_dup.pl
export CPERL='$USHlmp/lmp_scp_dup.pl'                 # 60 Character Length
export CINPUT='SCPEDIT_GOES${GOESreg}SCP.${HOURm1}'   # 60 Character Length
export CDATE1='date1'                                 # Earliest allowable date (60 Character Length)
export CDATE2='date2'                                 # Latest allowable date (60 Character Length)
startmsg
$EXEClmp/lmp_scpprep <<EOF >> $pgmout 2>errfile
$GOES ${premonchar}${monchar}${postmonchar}${day}${mndy}${year}
${DATmh1}
${CPERL}
${CINPUT}
${CDATE1}
${CDATE2}
EOF
# export err=$?; err_chk
# above export must be commented out in operations.  We do not want a 
# bad run of lmp_scpprep to kill the job since the SCP data is not 
# absolutely necessary; we can run without satellite data (and need to 
# do so during satellite outages or the eclipse periods. 

echo LMPLOG: `date` - end running lmp_scpprep.f

# Saving prepack file to unique name in temporary directory for purposes of
# troubleshooting if there were ever a problem. 
cp -p prepack.SCP.txt prepack.SCP.${GOESreg}.${HOURm1}.txt

#
#        NOTE REGARDING SCP DATA PROBLEMS:  IN THE EVENT OF A PROBLEM WITH THE SCP DATA, 
#        CREATING A ZERO-BYTE tcus_${GOESreg} FILE WILL ALLOW THE MAIN PROGRAM 
#        TO CONTINUE.  THE CODES THAT FOLLOW THIS ONE ARE PROGRAMMED TO BE ABLE 
#        TO HANDLE MISSING SCP DATA AND RELY ON NON-SCP-AUGMENTED OBS AS PREDICTORS. 
#

#
#        TREATMENT FOR prepack.SCP.txt WHICH HAS ONLY HEADER OR (HEADER AND ZZZZ)
#
count=`wc -l prepack.SCP.txt | awk '{ print $1 }'`
if (( $count <= 2 )); then
   rm -f lmp_sc${FILCOAST}_mdlp.${HOURm1}
   touch lmp_sc${FILCOAST}_mdlp.${HOURm1}  # 0 BYTE OUTPUT FILE
   echo "WARNING: No SCP data found in desired time window.  Zero-byte file created for tcus_${GOESreg} ${DATmh1}: EXITING SCP SCRIPT"
   exit 1    ##  YES! exit from script
fi

#
#        THE LMP_SCPPREP EXECUTABLE DID NOT COMPLETE. THEREFORE, USE THE
#        TOUCH COMMAND TO GENERATE A 0 BYTE PACKED FILE.
#
      if [[ ! -a "lmp_sc${FILCOAST}_mdlp.${HOURm1}" ]]; then
        touch lmp_sc${FILCOAST}_mdlp.${HOURm1}  # 0 BYTE OUTPUT FILE
        echo "WARNING: lmp_scppprep executable did not finish.  Zero-byte file created for tcus_${GOESreg} ${DATmh1}: EXITING SCP SCRIPT"
        exit 1    ##  YES! exit from script
      fi
#
echo "SUCCESSFUL COMPLETION OF SCP ${GOESreg} PROCESSING FOR $DATmh1"

echo LMPLOG: `date -u` - lmp_scpprep.sh ended
exit 0