#! /bin/bash ###----------------------------------------------------------------------------### ###--- ---### ###--- Script to generate a standard adeck input file for HCCA, DTOPS, etc. ---### ###--- It should be run prior to these models. ---### ###--- ---### ###--- Usage: ./merge_input_adecks.bash ${tcid} ${iprod} ---### ###--- ---### ###--- where: ${tcid} = al092018 , ---### ###--- ${iprod} = 0, for example. iprod=0 ---> quasiprod mode, ---### ###--- iprod=1 ---> prod mode ---### ###--- ---### ###--- VERSION: 1.0.1 ---### ###--- ---### ###--- Edit history: ---### ###--- Matt Onderlinde - June 2018 - wrote original code ---### ###--- Stephanie Stevenson - July 2018 - modified HWRF/HMON ---### ###--- UEMN/CTCX retrieval ---### ###--- Stephanie Stevenson - Sept 2018 - added own EMX tracker---### ###--- if not in ncep a-deck---### ###--- Stephanie Stevenson - Mar 2019 - changed dir paths to ---### ###--- include env vars from ---### ###--- main driver script ---### ###--- Apr 2019 - added iprod flag to ---### ###--- only merge CTCX/UEMN ---### ###--- for HCCA in quasi-prod---### ###--- (iprod=0) ---### ###--- Matt & Stephanie - Jun 2019 - fixed missing file errs, ---### ###--- get date from com file, ---### ###--- & update ncep_adeck path ---### ###--- Stephanie Stevenson - Oct 2021 - modified for WCOSS2 ---### ###--- Matt Onderlinde - Apr 2022 - Switched to new EMY-to-EMX---### ###--- retrieval method for early---### ###--- EMX tracker ---### ###--- Stephanie Stevenson - Feb 2023 - removed iprod flag ---### ###--- for inclusion of HCCA ---### ###--- and added CHIPS ---### ###--- Mar 2023 - replaced HWRF/HMON ---### ###--- with HAFS ---### ###--- May 2023 - added HWRF/HMON back ---### ###----------------------------------------------------------------------------### ###--- Define the paths to directory for: ###--- input b-deck file bdir=${bdir:-${atcfdir}/zfst/} ###--- com file directory comdir_merge=${atcfdir}/zcom/ ###--- input NCEP a-deck file #adir=${COMROOTp1}/nhc/prod/atcf/ adir=${TRACKglob}/atcf/ ###--- input HMON/HWRF a-deck file hwrfdir=${TRACKhwrf} hmondir=${TRACKhmon} ###--- input HAFS a-deck file hafsdir=${TRACKhafs} ###--- input UKMET ensemble a-deck file ukmet_path=${DCOMINnhc}/opah_adecks/ukmet/ ###--- input COAMPS a-deck file coamps_path=${DCOMINnhc}/opah_adecks/coamps/ ###--- input CHIPS a-deck file chips_path=${DCOMINnhc}/misc_adecks/chips/ ###--- Define login info for retrieving COAMPS and UKMET from OPAH #opahuser=nhcingest #opahserver=opah.ncep.noaa.gov ###--- Define paths to COAMPS and UKMET a-decks on OPAH server #opah_coamps_path=/home/people/nhc/users/nhcingest/incoming/trackers/COAMPS/ ### sample: /home/people/nhc/users/nhcingest/incoming/trackers/COAMPS/ep062018_2018062818.ctcx.dat #opah_ukmet_path=/home/people/nhc/users/nhcingest/incoming/ukmet/ ### sample: /home/people/nhc/users/nhcingest/incoming/ukmet/MOGREPSG_ADECK_2018062906.txt ###--- Define an egrep string that will catch all desired models from an a-deck ("|" delimited) amods="AEMN|AVNO|AVNX|CEMN|CHIP|CMC|CTCX|EEMN|EMX|FEMN|GFSO|HMON|HWRF|HFSB|HFSA|NGX|UKX|UEMN|GFDL|GFDT|NAM" ###--- Define output file name convention ( the actual file will be ${tcid}_ ) output_file_name_convention="interim_adeck.dat" ###--- Define directory for output file outputdir="${WORK_DIR}/storm-data" ###--- WORK_DIR is an environment variable ###----------------------------------------------------------------------------### ###----------------------------------------------------------------------------### ###--- Grab start time start=`date` ###--- Grab the iprod flag iprod=${2} #use iprod=1 for NCO-run version #iprod=1 ###--- Get input TCID tcid=${1} if [ ${#tcid} -eq 0 ];then echo ""; echo "ERROR: Must pass in TCID: ${0} al092018"; echo "" exit fi ###--- If this is a storm for which advisories are being issued, get the Invest number (from this storm's invest stage) ###--- and determine if the current synoptic time is within 18 hours of transition. basin=`echo ${tcid} | cut -c 1-2 | tr '[:upper:]' '[:lower:]'` if [ "${basin}" = "AL" ]; then basin_short=${basin:1:2}; else basin_short=${basin:0:1}; fi snum=` echo ${tcid} | cut -c 3-4` prev_snum=${snum} to_append_invest_stage=0 if [ `echo "${snum} < 49" | bc` -eq 1 ];then ###--- Ensure that bdeck actually exists if [ -f ${bdir}/b${tcid}.dat ];then ###--- Determine Invest-stage TCID (90s series) and transition time nineties_series_number=`grep -io "${basin}[A-Z][0-9]" ${bdir}/b${tcid}.dat | tail -1 | cut -f 2 -d " " | cut -c 4` if [ ${#nineties_series_number} -gt 0 ];then ###--- Get invest ID and transition synoptic time/epoch prev_tcid="${basin}9${nineties_series_number}`echo ${tcid} | cut -c 5-8`" transition_time=`grep -i "${basin}[A-Z][0-9]" ${bdir}/b${tcid}.dat | tail -1 | cut -f 3 -d "," | sed -e 's/ //g'` transition_epoch_seconds=`date -d "${transition_time:0:8} ${transition_time:8:2}:00:00" "+%s"` ###--- Determine most recent synoptic time most_recent_synoptic_time=`grep -o ", 2[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]," ${bdir}/b${tcid}.dat | tail -1 | cut -f 2 -d "," | sed -e 's/ //g'` most_recent_synoptic_epoch_seconds=`date -d "${most_recent_synoptic_time:0:8} ${most_recent_synoptic_time:8:2}:00:00" "+%s"` time_difference=$(( ( ${most_recent_synoptic_epoch_seconds} - ${transition_epoch_seconds} ) / 3600 )) ###--- If transition occurred during the previous 18 hours, grab Invest-stage a-deck lines if [ ${time_difference} -lt 19 ];then ###--- Generate random working file names to_append_invest_stage=1 randf1=${RANDOM}${RANDOM}${RANDOM}${RANDOM}.txt randf2=${RANDOM}${RANDOM}${RANDOM}${RANDOM}.txt randf3=${RANDOM}${RANDOM}${RANDOM}${RANDOM}.txt ###--- Define synoptic times surrounding the transition time check_synoptic_times="" hfsa_hfsb_files="" hwrf_hmon_files="" for n in $(seq -3 1 2) do cur_epoch=$(( ${transition_epoch_seconds} + ( ${n}*6*3600 ) )) cur_syn_time=`date -d @${cur_epoch} "+%Y%m%d%H"` check_synoptic_times="${check_synoptic_times}|${cur_syn_time}" hwrf_adeck=${hwrfdir}/hwrf.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${prev_tcid}/ncep_a${prev_tcid}.dat hwrf_adeck_cur=${hwrfdir}/hwrf.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hmon_adeck=${hmondir}/hmon.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${prev_tcid}/ncep_a${prev_tcid}.dat hmon_adeck_cur=${hmondir}/hmon.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hwrf_hmon_files="${hwrf_hmon_files} ${hwrf_adeck} ${hmon_adeck} ${hwrf_adeck_cur} ${hmon_adeck_cur}" hfsa_adeck=${hafsdir}/hfsa.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${prev_tcid}/ncep_a${prev_tcid}.dat hfsa_adeck_cur=${hafsdir}/hfsa.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hfsb_adeck=${hafsdir}/hfsb.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${prev_tcid}/ncep_a${prev_tcid}.dat hfsb_adeck_cur=${hafsdir}/hfsb.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hfsa_hfsb_files="${hfsa_hfsb_files} ${hfsa_adeck} ${hfsb_adeck} ${hfsa_adeck_cur} ${hfsb_adeck_cur}" done check_synoptic_times=`echo ${check_synoptic_times} | cut -f 2-100 -d "|"` ###--- Grab invest lines for cycles that are close-in-time to the transition time invest_adeck=${adir}/${prev_tcid}/ncep_a${prev_tcid}.dat egrep -h "(${check_synoptic_times})" ${invest_adeck} ${hwrf_hmon_files} ${hfsa_hfsb_files} | egrep "(${amods})" > ${randf1} ###--- Grab a-deck lines for first synoptic time in new storm a-deck grep "${transition_time}" ${adir}/${tcid}/ncep_a${tcid}.dat > ${randf2} grep "${transition_time}" ${hwrfdir}/hwrf.${transition_time:0:8}/${transition_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat >> ${randf2} grep "${transition_time}" ${hmondir}/hmon.${transition_time:0:8}/${transition_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat >> ${randf2} grep "${transition_time}" ${hafsdir}/hfsa.${transition_time:0:8}/${transition_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat >> ${randf2} grep "${transition_time}" ${hafsdir}/hfsb.${transition_time:0:8}/${transition_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat >> ${randf2} ###--- Only add invest lines that are unique orig_IFS=${IFS} prev_snum=`echo ${prev_tcid} | cut -c 3-4` for n in $(seq 1 1 `cat ${randf1} | wc -l`) do IFS="|" curline=`head -${n} ${randf1} | tail -1` curline_test=`echo ${curline} | cut -f 3-6 -d ","` if [ `grep "${curline_test}" ${randf2} | wc -l` -eq 0 ];then curline_part1=`echo ${curline} | cut -f 1-2 -d "," | sed -e "s/${prev_snum}/${snum}/g"` curline_part2=`echo ${curline} | cut -f 3-150 -d ","` echo "${curline_part1},${curline_part2}" >> ${randf3} fi IFS=${orig_IFS} done ###--- Create final merged a-deck and clean up temporary files cat ${randf2} ${randf3} ${adir}/${tcid}/ncep_a${tcid}.dat > ${tcid}_${output_file_name_convention} rm ${randf1} ${randf2} ${randf3} 2> /dev/null ###--- Otherwise just copy over the NCEP a-deck (in this case, the storm transition from invest more than 18 hours ago) else cp ${adir}/${tcid}/ncep_a${tcid}.dat ${tcid}_${output_file_name_convention} #--- Append all HAFS files for current and previous cycles hfsa_hfsb_files="" hwrf_hmon_files="" for n in $(seq -3 1 0) do cur_epoch=$(( ${most_recent_synoptic_epoch_seconds} + ( ${n}*6*3600 ) )) cur_syn_time=`date -d @${cur_epoch} "+%Y%m%d%H"` hwrf_adeck=${hwrfdir}/hwrf.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hmon_adeck=${hmondir}/hmon.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hwrf_hmon_files="${hwrf_hmon_files} ${hwrf_adeck} ${hmon_adeck}" hfsa_adeck=${hafsdir}/hfsa.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hfsb_adeck=${hafsdir}/hfsb.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hfsa_hfsb_files="${hfsa_hfsb_files} ${hfsa_adeck} ${hfsb_adeck}" done cat ${hwrf_hmon_files} ${hfsa_hfsb_files} >> ${tcid}_${output_file_name_convention} fi ###--- Otherwise just copy over the NCEP a-deck (this might occur if the storm never had an invest stage) else cp ${adir}/${tcid}/ncep_a${tcid}.dat ${tcid}_${output_file_name_convention} #--- Append all HAFS files for current and previous cycles hfsa_hfsb_files="" hwrf_hmon_files="" most_recent_synoptic_time=`grep -o ", 2[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]," ${bdir}/b${tcid}.dat | tail -1 | cut -f 2 -d "," | sed -e 's/ //g'` most_recent_synoptic_epoch_seconds=`date -d "${most_recent_synoptic_time:0:8} ${most_recent_synoptic_time:8:2}:00:00" "+%s"` for n in $(seq -3 1 0) do cur_epoch=$(( ${most_recent_synoptic_epoch_seconds} + ( ${n}*6*3600 ) )) cur_syn_time=`date -d @${cur_epoch} "+%Y%m%d%H"` hwrf_adeck=${hwrfdir}/hwrf.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hmon_adeck=${hmondir}/hmon.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hwrf_hmon_files="${hwrf_hmon_files} ${hwrf_adeck} ${hmon_adeck}" hfsa_adeck=${hafsdir}/hfsa.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hfsb_adeck=${hafsdir}/hfsb.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat hfsa_hfsb_files="${hfsa_hfsb_files} ${hfsa_adeck} ${hfsb_adeck}" done cat ${hwrf_hmon_files} ${hfsa_hfsb_files} >> ${tcid}_${output_file_name_convention} fi else echo "ERROR: Could not find a bdeck for ${tcid} !!!" fi ###--- Otherwise just copy over the NCEP a-deck (in this case, the storm is still an invest) else if [ -f ${adir}/${tcid}/ncep_a${tcid}.dat ];then cp ${adir}/${tcid}/ncep_a${tcid}.dat ${tcid}_${output_file_name_convention} fi #--- Append all HAFS files for current and previous cycles hfsa_hfsb_files="" hwrf_hmon_files="" most_recent_synoptic_time=`grep ", CARQ, 0," ${comdir_merge}/${tcid}.com | grep -o ", 2[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]," | tail -1 | cut -f 2 -d "," | sed -e 's/ //g'` most_recent_synoptic_epoch_seconds=`date -d "${most_recent_synoptic_time:0:8} ${most_recent_synoptic_time:8:2}:00:00" "+%s"` for n in $(seq -3 1 0) do cur_epoch=$(( ${most_recent_synoptic_epoch_seconds} + ( ${n}*6*3600 ) )) cur_syn_time=`date -d @${cur_epoch} "+%Y%m%d%H"` hwrf_adeck=${hwrfdir}/hwrf.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat echo ${hwrf_adeck} hmon_adeck=${hmondir}/hmon.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat if [ -f ${hwrf_adeck} ];then hwrf_hmon_files="${hwrf_hmon_files} ${hwrf_adeck}" fi if [ -f ${hmon_adeck} ];then hwrf_hmon_files="${hwrf_hmon_files} ${hmon_adeck}" fi hfsa_adeck=${hafsdir}/hfsa.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat echo ${hfsa_adeck} hfsb_adeck=${hafsdir}/hfsb.${cur_syn_time:0:8}/${cur_syn_time:8:2}/atcf/${tcid}/ncep_a${tcid}.dat if [ -f ${hfsa_adeck} ];then hfsa_hfsb_files="${hfsa_hfsb_files} ${hfsa_adeck}" fi if [ -f ${hfsb_adeck} ];then hfsa_hfsb_files="${hfsa_hfsb_files} ${hfsb_adeck}" fi done if [ ${#hwrf_hmon_files} -gt 0 ];then cat ${hwrf_hmon_files} >> ${tcid}_${output_file_name_convention} fi if [ ${#hfsa_hfsb_files} -gt 0 ];then cat ${hfsa_hfsb_files} >> ${tcid}_${output_file_name_convention} fi fi ###--- Get COAMPS and UKMET (ensemble mean) from OPAH files and CHIPS most_recent_synoptic_time=`grep ", CARQ, 0," ${comdir_merge}/${tcid}.com | grep -o ", 2[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]," | tail -1 | cut -f 2 -d "," | sed -e 's/ //g'` most_recent_synoptic_epoch_seconds=`date -d "${most_recent_synoptic_time:0:8} ${most_recent_synoptic_time:8:2}:00:00" "+%s"` basin_capitalized=`echo ${basin} | tr '[:lower:]' '[:upper:]'` randf4=${RANDOM}${RANDOM}${RANDOM}${RANDOM}.txt for n in $(seq -3 1 0) do echo "inside loop, n = ${n} ..." cur_epoch=$(( ${most_recent_synoptic_epoch_seconds} + ( ${n}*6*3600 ) )) cur_syn_time=`date -d @${cur_epoch} "+%Y%m%d%H"` cat ${coamps_path}/${tcid}_${cur_syn_time}.ctcx.dat ${ukmet_path}/MOGREPSG_ADECK_${cur_syn_time}.txt ${chips_path}/CHIPS.${cur_syn_time}_${snum}${basin_short^^}.xfer | egrep "(${amods})" | egrep "(${basin_capitalized}, ${snum},|${basin_capitalized}, ${prev_snum},)" >> ${randf4} if [ ${to_append_invest_stage} -eq 1 ];then cat ${coamps_path}/${prev_tcid}_${cur_syn_time}.ctcx.dat ${chips_path}/CHIPS.${cur_syn_time}_${prev_snum}${basin_short^^}.xfer | egrep "(${amods})" | egrep "(${basin_capitalized}, ${snum},|${basin_capitalized}, ${prev_snum},)" >> ${randf4} fi done ###--- Rename COAMPS/UKMET invest with current tcid if [ `echo "${snum} < 49" | bc` -eq 1 ];then orig_IFS=${IFS} randf4b=${RANDOM}${RANDOM}${RANDOM}${RANDOM}.txt prev_snum=`echo ${prev_tcid} | cut -c 3-4` for n in $(seq 1 1 `cat ${randf4} | wc -l`) do IFS="|" curline=`head -${n} ${randf4} | tail -1` curline_part1=`echo ${curline} | cut -f 1-2 -d "," | sed -e "s/${prev_snum}/${snum}/g"` curline_part2=`echo ${curline} | cut -f 3-150 -d ","` echo "${curline_part1},${curline_part2}" >> ${randf4b} IFS=${orig_IFS} done if [ -f ${randf4b} ];then cat ${randf4b} >> ${tcid}_${output_file_name_convention} rm ${randf4} ${randf4b} fi else cat ${randf4} >> ${tcid}_${output_file_name_convention} rm ${randf4} fi rm ${randf4} ###--- Check for EMX tracker for the current synoptic time ###--- if already merged from ncep_ a-deck, do nothing else ###--- if not and 180-h non-NCO-run EMX tracker ran, set EMY tracker to EMX for downstream code for n in $(seq -3 1 0) do cur_epoch=$(( ${most_recent_synoptic_epoch_seconds} + ( ${n}*6*3600 ) )) cur_syn_time=`date -d @${cur_epoch} "+%Y%m%d%H"` if [ ${cur_syn_time:(-2)} -eq 00 ] || [ ${cur_syn_time:(-2)} -eq 12 ]; then if [ -f ${adir}/${tcid}/ncep_a${tcid}.dat ];then emx_check=`grep "${cur_syn_time}" ${adir}/${tcid}/ncep_a${tcid}.dat | grep -c "EMX"` if [ ${emx_check} -eq 0 ];then grep ${cur_syn_time} ${adir}/${tcid}/ncep_a${tcid}.dat | egrep "(${basin_capitalized}, ${snum},|${basin_capitalized}, ${prev_snum},)" | \ grep EMY | sed -e 's/ EMY,/ EMX,/g' -e "s/${basin_capitalized}, ${prev_snum},/${basin_capitalized}, ${snum},/g" >> ${tcid}_${output_file_name_convention} fi fi fi done ###--- Sort final output a-deck by synoptic time if [ -f ${tcid}_${output_file_name_convention} ]; then randf5=${RANDOM}${RANDOM}${RANDOM}${RANDOM}.txt sort -t "," -k3,3n -k5,5d -k6,6n ${tcid}_${output_file_name_convention} > ${randf5} mv ${randf5} ${tcid}_${output_file_name_convention} fi ###--- Move to final location if [ -f ${tcid}_${output_file_name_convention} ]; then mv ${tcid}_${output_file_name_convention} ${outputdir}/${tcid}/ else touch ${outputdir}/${tcid}/${tcid}_${output_file_name_convention} fi if [ -f early_emh_tracker.tmp ]; then mv early_emh_tracker.tmp ${outputdir}/${tcid}/ fi ###--- Report some of the basic parameters finish=`date` echo "" echo "Runtime parameters for ${0} :" if [ ${to_append_invest_stage} -eq 1 ];then echo "prev_tcid = ${prev_tcid}" echo "transition_time = ${transition_time}" echo "most_recent_synoptic_time = ${most_recent_synoptic_time}" echo "time_difference = ${time_difference}" else echo "Current situation does not require appending of invest-stage atcf lines ..." if [ -f ${adir}/${tcid}/ncep_a${tcid}.dat ];then echo "Just copied ${adir}/${tcid}/ncep_a${tcid}.dat to ${tcid}_${output_file_name_convention} ..." else echo "ERROR: could not find ${adir}/${tcid}/ncep_a${tcid}.dat" echo " NOTE: for TCIDs greater than 49, input files such as ncep_a${tcid}.dat may not be available." fi fi echo "Start time for ${0} was: ${start}" echo "Finish time for ${0} was: ${finish}" echo ""