#! /usr/bin/env python ##@namespace scripts.exhwrf_para_archive # Generates an archive file from HWRF COM directory outputs. This # archive file can be on disk, or on the HPSS archiving system (via # the htar command). # # How this is done depends on the [config] section archive option, in # the HWRF configuration files: # # * archive=none --- no archiving is done. This script will exit # immediately without doing anything. # # * archive=disk:/path/to/archive.tar.gz --- make a gzipped tar file via # the tar -czf command, and place it on disk in a long-term storage area # # * archive=hpss:/path/to/archive.tar --- use the "htar" command to # place an archive on tape. # # * archive=hpsz:/path/to/archive.tar.gz --- a two-step process. The # first step uses tar -czf to create an on-disk archive in a staging # area. The second step copies that archive to tape using "hsi put" import sys, os, glob import produtil.setup, produtil.log, produtil.run, produtil.cd import hwrf_expt from produtil.log import postmsg, jlogger from produtil.run import batchexe, checkrun, run from produtil.cd import NamedDir def main_disk(): """!Main program for disk archiving. Creates an on-disk archiving for one of two cases: * disk:/path/to/archive.tar.gz --- generates an on-disk *.tar.gz archive in a long-term storage disk area * hpsz:/path/to/tape-archive.tar.gz --- generates an on-disk *.tar.gz archive in a temporary location so it can be copied to tape in a later step.""" postmsg('hwrf_archive disk step starting') conf=hwrf_expt.conf logger=conf.log('archive.disk') if conf.has_section('archive'): makethedir=conf.getbool('archive','mkdir',False) else: makethedir=False archive=conf.getloc('archive','NONE') if archive=='NONE': logger.info('No archive location specified. Exiting.') postmsg('hwrf_archive disk step has nothing to do when archiving is ' 'disabled.') return daonly=conf.getbool('archive','daonly',False) with NamedDir(conf.getdir('com')): if not daonly: flist=[ filename+'\n' for filename in glob.glob('*') ] else: flist=[ filename+'\n' for filename in glob.glob('*gsi*diag*') ] flist=flist+[ filename+'\n' for filename in glob.glob('*gsi*stdout*') ] STID=conf.getstr('config','STID') flist=flist+[ filename+'\n' for filename in glob.glob(STID.lower()+'.*') ] flist=flist+[ filename+'\n' for filename in glob.glob('*f00*.grb2') ] flist=flist+[ filename+'\n' for filename in glob.glob('*hwrfanl_*.grb2f00') ] flist=flist+[ filename+'\n' for filename in glob.glob('*hwrfges_*.grb2f00') ] flist=flist+[ filename+'\n' for filename in glob.glob('storm1.*') ] flist=flist+[ filename+'\n' for filename in glob.glob('*atcfunix*') ] HH=conf.getstr('config','HH') if conf.getbool('config','run_ensemble_da'): flist=flist+[ filename+'\n' for filename in glob.glob('*ensda_*.wrfinput*') ] flist.sort() files=''.join(flist) assert(len(files)>0) if archive.lower()=='none': postmsg('Archiving is disabled: archive=none') return elif archive[0:5]=='disk:': path=archive[5:] if makethedir: adir=os.path.dirname(path) if not os.path.exists(adir): produtil.fileop.makedirs(adir,logger=logger) flags='-cvp' if path[-3:]=='.gz' or path[-4:]=='.tgz': flags+='z' cmd=batchexe(conf.getexe('tar'))[flags+'f',path,'-T','-'] << files elif archive[0:5]=='hpss:': logger.info('HPSS archiving enabled.') logger.info('Nothing to do in the disk archiving step.') logger.info('Returning successfully after doing nothing.') postmsg('hwrf_archive disk step does nothing when using htar ' 'archives.') return elif archive[0:5]=='hpsz:': path=conf.strinterp('config','{WORKhwrf}/stage-archive.tar.gz') flags='-cvp' if path[-3:]=='.gz' or path[-4:]=='.tgz': flags+='z' cmd=batchexe(conf.getexe('tar'))[flags+'f',path,'-T','-'] << files else: jlogger.error('Ignoring invalid archive method %s in %s' %(archive[0:4],archive)) return checkrun(cmd,logger=logger) postmsg('hwrf_archive disk step completed') def main_tape(): """!Main program for tape archiving. Does one of two things: * hpss:/path/to/archive.tar --- will use the htar command to archive COM directory outputs * hpsz:/path/to/archive.tar.gz --- will copy a tar.gz file from a temporary area, made by the disk archiving step, to a tape destination using the "hsi put" command.""" postmsg('hwrf_archive tape step starting') conf=hwrf_expt.conf logger=conf.log('archive.tape') archive=conf.getloc('archive','NONE') if conf.has_section('archive'): makethedir=conf.getbool('archive','mkdir',False) else: makethedir=False if archive=='NONE': logger.info('No archive location specified. Exiting.') postmsg('hwrf_archive disk step has nothing to do when archiving is ' 'disabled.') return daonly=conf.getbool('archive','daonly',False) with NamedDir(conf.getdir('com')): if not daonly: flist=[ filename+'\n' for filename in glob.glob('*') ] else: flist=[ filename+'\n' for filename in glob.glob('*gsi*diag*') ] flist=flist+[ filename+'\n' for filename in glob.glob('*gsi*stdout*') ] STID=conf.getstr('config','STID') flist=flist+[ filename+'\n' for filename in glob.glob(STID.lower()+'.*') ] flist=flist+[ filename+'\n' for filename in glob.glob('*f00*.grb2') ] flist=flist+[ filename+'\n' for filename in glob.glob('*hwrfanl_*.grb2f00') ] flist=flist+[ filename+'\n' for filename in glob.glob('*hwrfges_*.grb2f00') ] flist=flist+[ filename+'\n' for filename in glob.glob('storm1.*') ] flist=flist+[ filename+'\n' for filename in glob.glob('*atcfunix*') ] HH=conf.getstr('config','HH') if conf.getbool('config','run_ensemble_da'): flist=flist+[ filename+'\n' for filename in glob.glob('*ensda_*.wrfinput*') ] flist.sort() files=''.join(flist) assert(len(files)>0) if archive.lower()=='none': postmsg('Archiving is disabled: archive=none') return elif archive[0:5]=='disk:': logger.info('Disk archiving enabled.') logger.info('Nothing to do in the HPSS archiving step.') logger.info('Returning successfully after doing nothing.') postmsg('hwrf_archive tape step does nothing when using disk ' 'archives.') return elif archive[0:5]!='hpss:' and archive[0:5]!='hpsz:': jlogger.error('Ignoring invalid archive method %s in %s' %(archive[0:4],archive)) return if makethedir: adir=os.path.dirname(archive[5:]) logger.info('%s: make this HPSS directory, even if it exists'%(adir,)) mcmd=batchexe(conf.getexe('hsi'))['-P','mkdir','-p',adir] run(mcmd,logger=logger) if archive[0:5]=='hpss:': path=archive[5:] flags='-cvp' cmd=batchexe(conf.getexe('htar'))[flags+'f',path,'-L','-'] << files elif archive[0:5]=='hpsz:': topath=archive[5:] frompath=conf.strinterp('config', '{WORKhwrf}/stage-archive.tar.gz') cmd=batchexe(conf.getexe('hsi'))['put',frompath,':',topath] checkrun(cmd,logger=logger) postmsg('hwrf_archive tape step completed') if __name__=='__main__': try: acase=os.environ.get('ARCHIVE_STEP','BOTH').upper() produtil.setup.setup() hwrf_expt.init_module() if acase == 'DISK': main_disk() elif acase == 'TAPE': main_tape() elif acase == 'BOTH': main_disk() main_tape() else: postmsg('INVALID JHWRF_ARCHIVE STEP %s!! ABORTING!' %(repr(acase),)) except Exception as e: jlogger.critical('hwrf_archive is aborting: '+str(e),exc_info=True) sys.exit(2)