# Program Name: nwm_grib2netcdf
# Author: Donald Johnson (donald.w.johnson@noaa.gov)
# Abstract: This program converts grib2 files produced by other models
#     and creates netcdf copies containing the variables needed by NWM
#     forcing engine
# Usage: python nwm_grib2netcdf.py
# Condition Codes

from pathlib import Path
import sys
import subprocess
import logging
import argparse
import os
from datetime import datetime
import time
import shutil
import re

seconds_in_a_day = 86400

logger = logging.getLogger('nwm_grb2netcdf')
logger.setLevel(logging.DEBUG)

log_handler = logging.FileHandler('nwm_grib2netcdf.log',mode='w')
log_handler.setLevel(logging.DEBUG)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log_handler.setFormatter(formatter)

logger.addHandler(log_handler)

grib2_products_global = ["GFS_Production_GRIB2", "GFS_Test", "NAM_Conus_Nest_GRIB2",
                         "HRRR_Conus_GRIB2", "RAP_Conus_GRIB2",
                         "CFSv2_6Hr_Global_GRIB2", "WRF_ARW_Hawaii_GRIB2",
                         "GFS_Production_025d_GRIB2", "NAM_Nest_3km_Hawaii",
                          "NAM_Nest_3km_PuertoRico", "NAM_Nest_3km_Alaska",
                          "MRMS_RadarOnly_QPE", "MRMS_RadarQualityIndex","MRMS_GaugeCorr_QPE",
                          "MRMS_Hawaii_MultiSensor", "MRMS_Hawaii_RadarQualityIndex",
                          "MRMS_Conus_Hawaii_MultiSensor_QPE",
                          "NAM_Nest_3km_Puerto_Rico","WRF_ARW_Puerto_Rico_GRIB2"]

arw_pattern = re.compile(".*\.(f..)\..+")
rap_conus_pattern = re.compile(".*z\.awp130bgrb(f..)\..+")
hrrr_conus_pattern = re.compile(".*z\.wrfsfc(f..)\..+")
gfs_conus_pattern = re.compile(".*z\.sfluxgrb(f.+)\..+")
nam_nest_pattern = re.compile(".*z\..+\.hires(f..)\..+")
cfs_pattern = re.compile(".*flxf(.+)\.[0-9][0-9]\.(.+)\..+")


# class for returning all file paths recursivly that match a pattern in target directory
class RecursiveFilePatternMatch:
    def __init__(self, basepath, pattern):
        self.basepath = basepath
        self.pattern = pattern

    def get_paths(self):
        search_path = Path(self.basepath)
        attempts = 0
        while attempts < 3:
            try:
                paths = search_path.glob(self.pattern)
                return list(paths)
            except FileNotFoundError:
                attempts = attempts + 1
                if attempts >= 3:
                    raise
            except:
                print("An unexpected error occured when searching the file system.")
                raise

# class for converting a grib2 file to a netcdf file wiith the required variable 
# for the NWM forcing engine

class ForcingNetcdfConverter:
    def __init__(self, product):
        self.gribvars = []
        self.netcdfvars = ['TMP_2maboveground','SPFH_2maboveground','UGRD_10maboveground'
                            'VGRD_10maboveground','PRATE_surface','DSWRF_surface','DLWRF_surface','PRES_surface'] 
        self.gribvars =   ['TMP','SPFH','UGRD','VGRD','PRATE','DSWRF','DLWRF','PRES'] 
        self.griblevels = ['2 m above ground','2 m above ground','10 m above ground','10 m above ground',
                           'surface','surface','surface','surface'] 
        self.product = product
        self.products = frozenset(grib2_products_global)


        # setup logging
        self.logger = logging.getLogger('ForcingNetcdfConverter')
        self.logger.setLevel(logging.DEBUG)
        self.logger.addHandler(log_handler)

    def warn(self,message):
        self.logger.warn(message)

    def error(self,message):
        self.logger.error(message)

    def critical(self,message):
        self.logger.critical(message)

    def info(self,message):
        self.logger.info(message)

    def debug(self,message):
        self.logger.debug(message)

    def convert(self, src_path, dst_path, sentinel_file):
        # get the parent of the distination and check that it exists
        pp = Path(dst_path).parent
        if not pp.exists():
            # make parents if necessary
            pp.mkdir(parents=True)
        
        if self.product in self.products:
            if self.product in ["MRMS_RadarOnly_QPE", "MRMS_RadarQualityIndex", "MRMS_GaugeCorr_QPE", "MRMS_Hawaii_MultiSensor", \
            "MRMS_Hawaii_RadarQualityIndex", "MRMS_Conus_Hawaii_MultiSensor_QPE"]:
                try:
                    # set up GRIB2TABLE if needed:
                    if not os.environ.get('GRIB2TABLE'):
                        g2path = os.path.join("grib2.tbl")
                        with open(g2path, 'wt') as g2t:
                            g2t.write(
                                "209:1:0:0:161:1:6:30:MultiSensorQPE01H:"
                                "Multi-sensor estimated precipitation accumulation 1-hour:mm\n"
                                 "209:1:0:0:161:1:6:37:MultiSensorQPE01H:"
                                 "Multi-sensor estimated precipitation accumulation 1-hour:mm\n"
                            )
                        os.environ['GRIB2TABLE'] = g2path
            
                    src_file = Path(src_path + ".gz")
                    dst_file = Path(dst_path)
                    #print("Source file:" + str(src_file))
                    #print("Destination file:" + str(dst_file))
                    tmp_file = dst_file.parent.joinpath(src_file.name)
                    #print("Temp file: " + str(tmp_file))

                    #make sure the dest directory exists
                    dst_file.parent.mkdir(parents=True, exist_ok=True)

                    #copy the src file to the tmp location
                    shutil.copy(str(src_file), str(tmp_file))
                
                    #unzip the compressed grib file
                    subprocess.run(["gunzip " + str(tmp_file)],shell=True)

                    #update the tmp_file name to match decompressed file
                    tmp_file = tmp_file.with_suffix('')

                    #use wgrib2 for conversion
                    cmd = "$WGRIB2 " + str(tmp_file) + " -netcdf " + str(dst_path[0:-3]) + ".nc"
                    self.info(self.product + " Running command: \"" + cmd +"\"")
                    
                    # call wgrib2 program
                    cmdOutput = subprocess.Popen([cmd],stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
                    out, err = cmdOutput.communicate()
                    exitcode = cmdOutput.returncode

                    if exitcode != 0:
                        print("FATAL ERROR: wgrib2 return non zero error code for file " + src_path)

                    sentinel_file.touch()
                    tmp_file.unlink()
                except:
                   self.info("Failed to find hour in : " + src_path)
            elif self.product in ["NAM_Nest_3km_Hawaii","NAM_Nest_3km_Puerto_Rico"]:
                try:
                    hr = int( nam_nest_pattern.match(src_path).groups()[0][1:] )
                    #command to extract need NWM variable this gets some extra variables
                    if ( 1 <= hr and hr <= 48 ):
                        cmd = "$WGRIB2 " +src_path + \
                            " -match \":(TMP):80 m above ground:|:(TMP|SPFH):2 m above ground:|:(UGRD|VGRD):10 m above ground:|:(PRES|APCP|HGT|PRATE|DSWRF|DLWRF):surface:\"" + \
                            " -netcdf " + dst_path
                        self.info(self.product + " Running command: \"" + cmd +"\"")

                        # call wgrib2 program
                        cmdOutput = subprocess.Popen([cmd],stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
                        out, err = cmdOutput.communicate()
                        exitcode = cmdOutput.returncode

                        if exitcode != 0:
                            print("FATAL ERROR: wgrib2 return non zero error code for file " + src_path)
                            Path(dst_path).unlink(missing_ok=True)
                        sentinel_file.touch()
                    else:
                        self.info("NAM hour not in range 1-48 found in " + src_path + " ignoring file.")
                except:
                    self.info("Failed to find hour in " + src_path)
            elif self.product in ["WRF_ARW_Hawaii_GRIB2","WRF_ARW_Puerto_Rico_GRIB2"]:
                # command to extract need NWM variable this gets some extra variables
                try:
                    hr = int( arw_pattern.match(src_path).groups()[0][1:] )
                    prevhr = hr - 1
                    if ( 1 <= hr and hr <= 48 ):
                        cmd = "$WGRIB2 " +src_path + \
                            " -match  ':(TMP):80 m above ground:" + str(hr) + " hour fcst:|" + \
                            ":(SPFH):2 m above ground:" + str(hr) + " hour fcst:|" + \
                            ":(UGRD|VGRD):10 m above ground:" + str(hr) +" hour fcst:|" + \
                            ":APCP:surface:" + str(prevhr) + "-"+ str(hr) + " hour acc fcst:|" + \
                            ":PRES:surface:" + str(hr) + " hour fcst:|:(HGT):surface:'" + \
                            " -netcdf " + dst_path
                        self.info(self.product + " Running command: \"" + cmd +"\"")

                        # call wgrib2 program
                        cmdOutput = subprocess.Popen([cmd],stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
                        out, err = cmdOutput.communicate()
                        exitcode = cmdOutput.returncode

                        if exitcode != 0:
                            print("FATAL ERROR: wgrib2 return non zero error code for file " + src_path)
                            Path(dst_path).unlink(missing_ok=True)
                        sentinel_file.touch()
                    else:
                        self.info("ARW hour not in range 1-48 found in " + src_path + " ignoring file.")
                except:
                    self.info("Failed to find hour in " + src_path)
            elif self.product in ["HRRR_Conus_GRIB2"]:
               # command to extract need NWM variables
                try:
                    hr = int( hrrr_conus_pattern.match(src_path).groups()[0][1:] )
                    prevhr = hr - 1
                    if ( 1 <= hr and hr <= 18 ):
                        cmd = "$WGRIB2 " +src_path + \
                             " -match ':(TMP|SPFH):2 m above ground:" + str(hr) +" hour fcst:|" + \
                             ":(UGRD|VGRD):10 m above ground:"+ str(hr) + " hour fcst:|" + \
                             ":(PRES|DLWRF|DSWRF):surface:" + str(hr) + " hour fcst:|" + \
                             ":APCP:surface:" + str(prevhr) + "-" + str(hr) + " hour acc fcst:|" + \
                             ":HGT:surface:" + str(hr) + " hour fcst:' -netcdf " + dst_path
                        self.info(self.product + " Running command: \"" + cmd +"\"")

                        # call wgrib2 program$$
                        cmdOutput = subprocess.Popen([cmd],stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
                        out, err = cmdOutput.communicate()
                        exitcode = cmdOutput.returncode

                        if exitcode != 0:
                            print("FATAL ERROR: wgrib2 return non zero error code for file " + src_path)
                            Path(dst_path).unlink(missing_ok=True)
                        sentinel_file.touch()
                    else:
                        self.info("HRRR outside the range 1-18 found in " + src_path + " ignoring file.")
                except:
                    self.info("Failed to find hour in " + src_path)
            elif self.product in ["RAP_Conus_GRIB2"]:
               # command to extract need NWM variables
                try:
                    hr = int( rap_conus_pattern.match(src_path).groups()[0][1:] )
                    prevhr = hr - 1
                    if ( 1 <= hr and hr <= 18 ):
                        cmd = "$WGRIB2 " +src_path + \
                             " -match ':(TMP|SPFH):2 m above ground:" + str(hr) +" hour fcst:|" + \
                             ":(UGRD|VGRD):10 m above ground:"+ str(hr) + " hour fcst:|" + \
                             ":(PRES|DLWRF|DSWRF):surface:" + str(hr) + " hour fcst:|" + \
                             ":APCP:surface:" + str(prevhr) + "-" + str(hr) + " hour acc fcst:|" + \
                             ":HGT:surface:" + str(hr) + " hour fcst:' -netcdf " + dst_path
                        self.info(self.product + " Running command: \"" + cmd +"\"")

                        # call wgrib2 program
                        cmdOutput = subprocess.Popen([cmd],stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
                        out, err = cmdOutput.communicate()
                        exitcode = cmdOutput.returncode

                        if exitcode != 0:
                            print("FATAL ERROR: wgrib2 return non zero error code for file " + src_path)
                            Path(dst_path).unlink(missing_ok=True)
                        sentinel_file.touch()
                    else:
                        self.info("RAP hour outside the range of 1-18 found in " + src_path + " ignoring file.")
                except:
                    self.info("Failed to find hour in " + src_path)
            elif self.product in ["GFS_Production_025d_GRIB2"]:
                try:
                    hr = int( gfs_conus_pattern.match(src_path).groups()[0][1:] )
                    prevhr = hr - 1
                    if ( 1 <= hr and hr <= 240 ):
                        if ( hr <= 120 or hr % 3 == 0 ):
                            # command to extract need NWM variable this gets some extra variables
                            cmd = "$WGRIB2 " +src_path + \
                                " -match \":(TMP|SPFH):2 m above ground:|:(UGRD|VGRD):10 m above ground:|:(PRES|PRATE|DLWRF|DSWRF|HGT|APCP):surface:\"" + \
                                " -netcdf " + dst_path

                            # call wgrib2 program
                            cmdOutput = subprocess.Popen([cmd],stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
                            out, err = cmdOutput.communicate()
                            exitcode = cmdOutput.returncode

                            if exitcode != 0:
                                print("FATAL ERROR: wgrib2 return non zero error code for file " + src_path)
                                Path(dst_path).unlink(missing_ok=True)
                            sentinel_file.touch()
                        else:
                            self.info("GFS hour greater than 120 found that is not divisible by 3 in " + src_path + "ignorning file.")
                    else:
                        self.info("GFS hour greater than 240 found in " + src_path + " ignoring file.")
                except:
                    self.info("Failed to find hour in " + src_path)
            elif self.product in ["CFSv2_6Hr_Global_GRIB2"]:
                try:
                    # test for regexp match and then extract CFS date strings from path
                    match = cfs_pattern.match(src_path)

                    #make datetime objects and get timedelta
                    dt1 = datetime.strptime(match.groups()[0],'%Y%m%d%H')
                    dt2 = datetime.strptime(match.groups()[1],'%Y%m%d%H')

                    if ( (dt1 - dt2).days <= 30 ):
                        # command to extract need NWM variable this gets some extra variables
                        cmd = "$WGRIB2 " +src_path + \
                            " -match \":(TMP|SPFH):2 m above ground:|:(UGRD|VGRD):10 m above ground:|:(PRES|PRATE|DLWRF|DSWRF|HGT|APCP):surface:\"" + \
                            " -netcdf " + dst_path

                        # call wgrib2 program
                        cmdOutput = subprocess.Popen([cmd],stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
                        out, err = cmdOutput.communicate()
                        exitcode = cmdOutput.returncode

                        if exitcode != 0:
                            print("FATAL ERROR: wgrib2 return non zero error code for file " + src_path)
                            Path(dst_path).unlink(missing_ok=True)
                        sentinel_file.touch()

                    else:
                        self.info("CFS dates found in " + src_path + " are greateer than 30 days apart. Ignoring this file. ")
                except:
                    self.info("Failed to parse CFS forecast dates.")
            else:
                # command to extract need NWM variable this gets some extra variables
                cmd = "$WGRIB2 " +src_path + \
                    " -match \":(TMP|SPFH):2 m above ground:|:(UGRD|VGRD):10 m above ground:|:(PRES|PRATE|DLWRF|DSWRF|HGT|APCP):surface:\"" + \
                    " -netcdf " + dst_path

                # call wgrib2 program$$$
                cmdOutput = subprocess.Popen([cmd],stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
                out, err = cmdOutput.communicate()
                exitcode = cmdOutput.returncode

                if exitcode != 0:
                    print("FATAL ERROR: wgrib2 return non zero error code for file " + src_path)
                    Path(dst_path).unlink(missing_ok=True)
                sentinel_file.touch()

        else:
            #unexpect product
            self.error("Conversion requested for unknown product: " + self.product)

        return

    
# function to return the differnce between two lists
def diff(lst1, lst2): 
    temp = set(lst2) 
    lst3 = [value for value in lst1 if value not in temp] 
    return lst3 

def main():

    # get the command line parser
    parser = argparse.ArgumentParser(description='Convert GRIB2 model output files to netcdf for ingest by NWM forcing engine.')
    parser.add_argument('-i', '--input-dir',  required=True, type=str, help='The root directory to search for input files')
    parser.add_argument('-o', '--output-dir', required=True, type=str, help='The root directory for converted files')
    parser.add_argument('-p', '--product',    
                        required=True, 
                        type=str, 
                        choices=grib2_products_global, 
                        help='The product to find input files for')
    parser.add_argument('-t', '--temp-dir', required=True, help='The location to store temporary files')
    args = parser.parse_args()

    #gribpath = '/gpfs/dell1/nco/ops/com/gfs/prod'
    #netcdfpath = 'converted'

    gribpath = args.input_dir
    netcdfpath = args.output_dir
    product = args.product
    temp_dir = args.temp_dir
    
    # pattern to match all the input and output files

    input_pattern = {
        'GFS_Production_025d_GRIB2' : 'gfs.????????/??/atmos/gfs.*sfluxgrbf*.grib2.idx',
	    'GFS_Test' : 'gfs.*/*/atmos/gfs.*sfluxgrbf*.grib2.idx',
        'HRRR_Conus_GRIB2' : 'hrrr.????????/conus/hrrr.t??z.*wrfsfcf*.grib2.idx',
        'CFSv2_6Hr_Global_GRIB2' : 'cfs.????????/*/6hrly_grib*/flxf*.grb2.idx',
        'RAP_Conus_GRIB2' : 'rap.????????/rap.*z.awp130bgrb*.grib2.idx',
        'MRMS_RadarOnly_QPE' : 'conus/RadarOnly_QPE/RadarOnly_QPE_01H_00.00_*0000.grib2.gz',
        'MRMS_RadarQualityIndex' : '**/RadarQualityIndex/RadarQualityIndex_00.00_*0000.grib2.gz',
        'MRMS_GaugeCorr_QPE' : '**/GaugeCorr_QPE/GaugeCorr_QPE_01H_00.00_*0000.grib2.gz',
        'WRF_ARW_Hawaii_GRIB2' : 'hiresw.????????/hiresw.t*z.arw_2p5km.f[01234]?.hi.grib2.idx',
        'NAM_Nest_3km_Hawaii' : 'nam.????????/nam.t*z.hawaiinest.hiresf[01234]?.tm00.grib2.idx',
        'MRMS_Hawaii_MultiSensor' : '**/MultiSensor/*/MRMS_EXP_MultiSensor_QPE_??H_Pass1_00.00_*-??0000.grib2.gz',
        'MRMS_Hawaii_RadarQualityIndex' : '**/RadarQualityIndex/*/MRMS_EXP_RadarQualityIndex_00.00_*-??0000.grib2.gz',
        'NAM_Nest_3km_Puerto_Rico' : 'nam.????????/nam.t*z.priconest.hiresf[01234]?.tm00.grib2.idx',
        'WRF_ARW_Puerto_Rico_GRIB2' : 'hiresw.????????/hiresw.t*z.arw_2p5km.f[01234]?.pr.grib2.idx',
        'MRMS_Conus_Hawaii_MultiSensor_QPE' : '[c,h][o,a][n,w]*/MultiSensorQPE/MRMS_MultiSensor_QPE_01H_Pass?_00.00_*.grib2.gz'
        }
        
    output_pattern = { 
        'GFS_Production_025d_GRIB2' : 'gfs.????????/??/atmos/gfs.*sfluxgrbf*.nc',
		'GFS_Test' : '/gfs.*/*/atmos/gfs.*sfluxgrbf*.nc',
        'HRRR_Conus_GRIB2' : 'hrrr.????????/conus/hrrr.*wrfsfcf*.nc',
        'CFSv2_6Hr_Global_GRIB2' : 'cfs.????????/*/6hrly_grib*/flxf*.nc',
        'RAP_Conus_GRIB2' : 'rap.????????/rap.*z.awp130bgrb*.nc',
        'MRMS_RadarOnly_QPE' : 'conus/RadarOnly_QPE/RadarOnly_QPE_01H_00.00_*00.nc',
        'MRMS_RadarQualityIndex' : '/RadarQualityIndex/RadarQualityIndex_00.00_*00.nc',
        'MRMS_GaugeCorr_QPE' : '/GaugeCorr_QPE/GaugeCorr_QPE_01H_00.00_*00.nc',
        'WRF_ARW_Hawaii_GRIB2' : 'hiresw.????????/hiresw.t*z.arw_2p5km.f*.hi.nc',
        'NAM_Nest_3km_Hawaii' : 'nam.????????/nam.t*z.hawaiinest.hiresf*.tm00.nc',
        'MRMS_Hawaii_MultiSensor' : '/MultiSensor/*/MRMS_EXP_MultiSensor_QPE_??H_Pass1_00.00_*-??0000.nc',
        'MRMS_Hawaii_RadarQualityIndex' : '/RadarQualityIndex/*/MRMS_EXP_RadarQualityIndex_00.00_*-??0000.nc',
        'NAM_Nest_3km_Puerto_Rico' : 'nam.????????/nam.t*z.priconest.hiresf[01234]?.tm00.nc',
        'WRF_ARW_Puerto_Rico_GRIB2' : 'hiresw.????????/hiresw.t*z.arw_2p5km.f[01234]?.pr.nc',
        'MRMS_Conus_Hawaii_MultiSensor_QPE' : '[c,h][o,a][n,w]*/MultiSensorQPE/MRMS_MultiSensor_QPE_01H_Pass?_00.00_*.nc'
        }

    pattern1 = input_pattern[product]
    pattern2 = output_pattern[product]

    # check for sentinel value
    sentinel_file = Path(temp_dir) / (product+'.sentinel')
    if sentinel_file.exists():
        lockfile_update_time = os.path.getmtime(str(sentinel_file))
        ct = time.time()
        if ct - lockfile_update_time < 5 * 60 :
            logger.info("Current Lock file detected, terminating process.")
            return 1
        else:
            logger.info("Old lock file located, ignoring lock file.")
            sentinel_file.touch()
    else:
        sentinel_file.touch()

    #sometimes date arrives late so check for files twice once at startup and once after the files
    # found at startup have been processed
    for run in ["main","failsafe"]:
    
        # make the pattern matching files
        pm1 = RecursiveFilePatternMatch(gribpath,pattern1)

        #print(pattern1)

        # get the paths of sflux files with and index file
        # the index file is only created after the full file is downloaded
        target_file_paths = pm1.get_paths()

        #print(target_file_paths)

        #remove the grib2.index from the end of each path
        #change paths to be relative to the search directory and remove .grib2.index
        target_file_paths = [path.relative_to(gribpath).with_suffix('').with_suffix('') for path in target_file_paths]
        logger.info("Found " + str(len(target_file_paths)) + " conversion targets ")

        #now we need get the list of converted files
        nc_p = Path(netcdfpath)

        # if the output path does not exist creat it
        if nc_p.exists() == False:
            logger.info("Creating output path: " + str(nc_p))
            nc_p.mkdir()

        # list the files that have been converted
        pm2 = RecursiveFilePatternMatch(netcdfpath,pattern2)
        converted_paths = pm2.get_paths()
        logger.info("Found " + str(len(converted_paths)) + " already converted files")

        #remove the .netcdf so that the file name are comparable
        converted_paths = [path.relative_to(netcdfpath).with_suffix('') for path in converted_paths]


        #print(target_file_paths)
        #print(converted_paths)

        #calculate the files that in the prod directories but not converted
        to_convert = diff(target_file_paths, converted_paths)
        logger.info(str(len(to_convert)) + " files to convert")

        #calculate the file that are converted but are no longer in prod
        to_remove = diff(converted_paths, target_file_paths)
        logger.info(str(len(to_remove)) + " files to remove")

        # convert each grib2 file

        netcdf_converter = ForcingNetcdfConverter(product)
        for path in to_convert:
            src_path = Path(gribpath).joinpath(path)
            dst_path = Path(netcdfpath).joinpath(path)

            # try to read the maximum age in days of a file to process from the enviorment variable 'PRECONVERSION_MAX_DAYS'
            # if it does not exist or can not be read set this value to 2
            try:
                max_days_str = os.getenv("PRECONVERSION_MAX_DAYS")
                if max_days_str is None or max_days_str == "":
                    max_days = 2
                else:
                    max_days = int(max_days_str)
            except:
                max_days = 2
            
            try:
                # check modification date to see if we want to copy
                if product in ["MRMS_RadarOnly_QPE", "MRMS_RadarQualityIndex","MRMS_GaugeCorr_QPE","MRMS_Hawaii_MultiSensor",\
                	"MRMS_Hawaii_RadarQualityIndex","MRMS_Conus_Hawaii_MultiSensor_QPE"]:
                    tpath = str(src_path)+".grib2.gz"
                elif product in ["CFSv2_6Hr_Global_GRIB2"]:
                    tpath = str(src_path)+".grb2"
                else:
                    tpath = str(src_path)+".grib2"
                
                file_mod_time = os.path.getmtime(tpath)
                current_time = time.time()

                # check file modification file to ignore old data
                if ( current_time - file_mod_time < max_days * seconds_in_a_day):
            
                    # print number of days old file is for debug
                    print((current_time - file_mod_time) / seconds_in_a_day)

                    # convert src_path to dest path
                    if product in ["CFSv2_6Hr_Global_GRIB2"]:
                        logger.info("Converting "+str(src_path)+".grb2 to "+str(dst_path)+".nc")
                        print("Converting "+str(src_path)+".grb2 to "+str(dst_path)+".nc")
                        netcdf_converter.convert(str(src_path)+".grb2",str(dst_path)+".nc",sentinel_file)
                    else:
                        logger.info("Converting "+str(src_path)+".grib2 to "+str(dst_path)+".nc")
                        print("Converting "+str(src_path)+".grib2 to "+str(dst_path)+".nc")
                        netcdf_converter.convert(str(src_path)+".grib2",str(dst_path)+".nc",sentinel_file)
                else:
                    # print files that are skipped
                    # print("old Path: " + str(tpath) + " skipping")
                    pass
            except Exception as ex:
                import traceback
                print(ex)
                traceback.print_exc()
                print("unable to check modification date for " + str(tpath))
    
        logger.info("Done converting files")

        # remove out of date files
        for path in to_remove:
            #print(path)
            if path.exists():
                logger.info("Removing "+str(path))
                path.unlink()
        logger.info("Done removing files")
    # end for run loop

    #close the log file
    log_handler.close()
    
    #remove sentinel
    sentinel_file.unlink()
    return    

if __name__ == "__main__":
    main()