#!/usr/bin/perl

use FindBin qw($Bin);
# use lib "/usr/lib/perl5/vendor_perl/5.26.1";
#use Date::Manip;
use Time::Piece;

# module load pm5/1.0
# The following command identifies the local working directory
# for this program and fills this into the Bin variable.

$DEBUG_OUTPUT = 0;

##PROGRAM DESCRIPTION##########################################################
# Purpose:
#  To generate a text, space delimited, job timing table that is 
#  equivalent to the layout of the current job timing table created
#  by earlier programs.
#
#
# Command line variables:
#
# Input:
#  Content of the /ecf/rundir/ecf.log file. 
#LOG:[07:05:05 19.2.2013]  submitted: /prod00/rtofs00/jrtofs_global_analysis_pre
#MSG:[07:05:05 19.2.2013] --msg=/prod00/rtofs00/jrtofs_global_analysis_pre:ibmsp:submitting :ecfprod
#MSG:[07:05:05 19.2.2013] --alter add variable ECF_QID 820571 /prod00/rtofs00/jrtofs_global_analysis_pre  :ecfprod
#MSG:[07:05:05 19.2.2013] --alter add variable ECF_SUB_HOST 192.58.1.51 /prod00/rtofs00/jrtofs_global_analysis_pre  :ecfprod
#MSG:[07:05:05 19.2.2013] --msg=/prod00/rtofs00/jrtofs_global_analysis_pre:ibmsp:submit successful :ecfprod
#MSG:[07:05:07 19.2.2013] chd:init /prod00/rtofs00/jrtofs_global_analysis_pre
#LOG:[07:05:07 19.2.2013]  active: /prod00/rtofs00/jrtofs_global_analysis_pre
#MSG:[07:19:13 19.2.2013] --alter delete variable ECF_QID /prod00/rtofs00/jrtofs_global_analysis_pre  :nwprod
#MSG:[07:19:13 19.2.2013] --alter delete variable ECF_SUB_HOST /prod00/rtofs00/jrtofs_global_analysis_pre  :nwprod
#MSG:[07:19:13 19.2.2013] chd:complete /prod00/rtofs00/jrtofs_global_analysis_pre
#LOG:[07:19:13 19.2.2013]  complete: /prod00/rtofs00/jrtofs_global_analysis_pre

###### OLD SMS FORMAT BELOW ##############################################################
# # LOG:[23:55:54 15.3.2004] submitted:/prod12/fnlibm18/jfnl18_dump_alert
# # MSG:[23:55:55 15.3.2004] /prod12/fnlibm18/jfnl18_dump_alert:message form task
# # MSG:[23:55:55 15.3.2004] ::/prod12/fnlibm18/jfnl18_dump_alert:ibmsp:submitting
# # MSG:[23:55:56 15.3.2004] /prod12/fnlibm18/jfnl18_dump_alert:message form task
# # MSG:[23:55:56 15.3.2004] ::/prod12/fnlibm18/jfnl18_dump_alert:ibmsp:submit successful
# # LOG:[23:55:59 15.3.2004] active:/prod12/fnlibm18/jfnl18_dump_alert
# # LOG:[23:56:00 15.3.2004] complete:/prod12/fnlibm18/jfnl18_dump_alert
##### OLD SMS FORMAT END #################################################################
#
#   There is a sequence of 11 input records, when a job runs properly,
#  that produce one output record. The sequence is shown above. Notice
#  that a "LOG:[23:55:54 15.3.2004] submitted:" input record starts
#  the job submission process. Then 5 "MSG:" records log the interaction
#  of handshaking between ECF and the supercomputer. Then a "LOG:"
#  entry indicates the job is active, according to ECF.
#
#
# Output:
#  Completed jobs runtime log file name YYYYMMDD.daily in the /com/logs/runtime2
#  directory on the production machine.
#  Below is an example of the output record format for records with a status
#  of complete:
#  'ECF Job path name''Submitted Date/Time''Active Date/Time''Complete Date/Time'
#                                  YYYYMMDDhhmmss       YYYYMMDDhhmmss       YYYYMMDDhhmmss
#  <J>/utility/jcheck_config</J><S>19700101000000</S><A>20050206235611</A><C>20050206235612</C>
#  <J>/prod18/gdas18/dump/jgdas_dump_post_18</J><S>20050206235613</S><A>20050206235620</A><C>20050206235735</C>
#  <J>/job_view/jobview_prod</J><S>20050206235616</S><A>20050206235622</A><C>20050206235650</C>
#  <J>/prod00/arkv00/arkv_ncar</J><S>20050206235613</S><A>20050206235622</A><C>20050206235640</C>
#
#  There are 4 files created to enable communications between seperate runs
#  of this program. These files are stored in the /com/logs/runtime2/interProcess
#  subdirectory. Following is a description:
#   YYYYMMDD.compB4Active -- All jobs that have a time for completion before
#    the active state was found within the ecf.log file.
#    'Job/Suite Name''Completed time in seconds since 1970''Line number in log file'
#    <J>/transfer2</J><C>1107735186</C><L>1393</L>
#    <J>/prod00/arkv00/run_history/jrun_history00</J><C>1107737161</C><L>5174</L>
#    <J>/test/prod06/gfs06/mos/jgfs_extcoopmos_06</J><C>1107787281</C><L>102825</L>
#    <J>/dev12/ingest12/ssmi</J><C>1107795452</C><L>118573</L>
# 
#
#   prev_ECF_Submitted.txt -- Contains those jobs that have a submitted time
#    in seconds since 1970 but have not reached the active state within the
#    records that were processed in the current run of the program. The content
#    of this file is not formatted with XML since its content is used for
#    internal processing and contains only 2 components in each record:
#    The job name and time in seconds since 1970.
#    /prod18/gdas18/dump/jgdas_dump_18||1107820224
#    /prod18/ndas00||1107816871
#    /prod18/ndas00/time_minus_03hr||1107819841
#    /prod18/ndas00/time_minus_03hr/jndas_post_tm03_00||1107819841
#    /prod18/sref21||1107820298
#    /prod18/sref21/jsref_prep_21||1107820298
#
#
#   prev_ECF_Active.txt -- Contains those jobs that have a submitted time in
#    seconds since 1970 but have not reached the complete state within the
#    records that were processed in the current run of the program. The content
#    of this file is not formatted with XML since its content is used for
#    internal processing and contains only 2 components in each record:
#    The job name and time in seconds since 1970.
#    /prod00/sfcruc00||1107817307
#    /prod12||1107818465
#    /prod18||1107818278
#    /prod18/gdas18||1107820238
#    /prod18/gdas18/dump||1107820238
#    /prod18/gdas18/dump/jgdas_dump_18||1107820238
#
#
#   prevRunECFLineNumber.txt -- Contains only one line that holds number of the
#    line that was processed during the previous run of this program. 
#    NOTE: the starting line number is zero (0) rather than one.
#
#
#   YYYYMMDD.dropped -- All jobs that have had the processed altered in some fashion
#    <N>/para06</N><I>aborted</I><S>1108063658</S><A>1108063664</A><R>1108063679</R><L>130504</L>
#    <N>/para06/tracker06/jtracker_ngps_09</N><I>aborted</I><S>1108063828</S><A>1108063839</A><R>1108063841</R><L>130899</L>
#    <N>/para06/tracker06</N><I>aborted</I><S>1108063828</S><A>1108063839</A><R>1108063841</R><L>130900</L>
#    <N>/para06/aqm06/jaqm_post2_06</N><I>force</I><S></S><A></A><R>1108074065</R><L>149368</L>
#    <N>/para06/aqm06/jaqm_post1_06</N><I>force</I><S></S><A></A><R>1108074070</R><L>149370</L>
#
###############################################################################
##PSEUDO CODE##################################################################
#
# Check for instance of this program that may still be running.
#  Don't run multiple instances of this program since it produces
#  temp files that might cause conflict between multiple instances.
# Setup global variables to use throughout the run of this program.
#  Try to keep the assignments in one area.
# Grab any incomplete job sequence information from temp files:
#  There are 2 files that contain the job sequence information.
#  These files have only 2 fields, the job name and the seconds since 1970
#  when the job went into either of the submitted or active state.
# Determine where to start reading the input file (which line number).
# Loop - process each line of input file
#  Place each input record content into the appropriate hash.
#  If an abort occurs remove job name instance from all hashes and
#  write the job name into an aborted table file.
#  Once a job is marked complete write a record to the output file and
#  remove all instance of the job name from all hashes.
#   TODO:
#   Within the loop determine a /suite/family/job that is time dependent
#   has run within a window of the start time designated in a start
#   time table.
#
###############################################################################

$DATA = $ENV{'DATA'};
$PDY = $ENV{'PDY'};
$PDYm1 = $ENV{'PDYm1'};
$COMOUT = $ENV{'COMOUT'};
$COMOUTruntime = $ENV{'COMOUTruntime'};
$COMOUTruntime2 = $ENV{'COMOUTruntime2'};
$COMOUTdaily = $ENV{'COMOUTdaily'};
# JY $COMOUT2 = $ENV{'COMOUT2'};
$dataDir = $ENV{DATA};
$interProcDir = $ENV{COMOUTinter};
$theECFMachine = $ENV{ECF_HOST};
$progName = $ENV{'job'};
#$DEBUG_OUTPUT = $ENV{DEBUG_OUTPUT};
$COMIN_ECFLOG=$ENV{COMIN_ECFLOG};

$ave30dfull = "$COMOUT/ave30/ave30day.full";
$ave30day = "$COMOUT/ave30/ave30day.short";

#
# Define 30-day ave all hash tables
#
%a30start = ();
%a30stop = ();
%start30 = ();
%stop30 = ();
%job30 = ();
%new30start = ();
%new30stop = ();

$previousECFMachineFile = $interProcDir . "/" . "prev_Machine.txt";
$thePrevMachine = `cat $previousECFMachineFile`;

######################################################################
# If the server has changed pull the ecf.log file from the previous server
# This helps ensure the log file from the previous server is completely processed.
######################################################################
$notMachineChanged = 1;
print "DEBUG: thePrevMachine=$thePrevMachine, theECFMachine=$theECFMachine\n";
if( $thePrevMachine ne $theECFMachine ){
  open( $PMH , ">$previousECFMachineFile" ) or warn "Couldn't open $previousECFMachineFile for update.\n";
  print $PMH $theECFMachine;
  close( $PMH );
  $theECFMachine = $thePrevMachine;
  # JY $copyString = "-q ecfprod\@$theECFMachine:/ecf/rundir/ecf.$theECFMachine.log $dataDir/ecf.log 2>&1 > /dev/null";
  # system( "/usr/bin/scp $copyString" );
  $copyString = "-p $COMIN_ECFLOG/ecf.$theECFMachine.log $dataDir/ecf.log 2>&1 > /dev/null";
  print "DEBUG1: copyString=$copyString, theECFMachine=$theECFMachine \n";
  system( "/usr/bin/cp $copyString" );
  warn "Detected machine switch. Will process final entries from $theECFMachine log file.\n";
  $notMachineChanged = 0;
} else {
  $thePrevMachine = $theECFMachine;
#  warn "Using $theECFMachine ecf.log file.\n";
}

if( $DEBUG_OUTPUT == 0 ){
  # During operations the crontab assigns a full path to the dollarZero variable
  #$thisProgIsRunningFile = "$0" . "_isRunning";
  $thisProgIsRunningFile = $interProcDir . "/" . $progName . "_isRunning";
} else {
  $thisProgIsRunningFile = $progName . "_isRunning";
}
$progPID = $$;
$theCurrentTime = time();
$maxTime2Run = 800;

###############################################################################
# Program instance checking
# Determine if this program is already running and if so don't continue unless
# the previous instance has been running too long.
# NOTE: I realized that it was better to use the time in the isRunning file than
#       use the process ID since there was no guarantee that a new instance
#       of the program would be run on the same node as the previous run. So
#       the only way to ensure there is no, or should be no previous instance,
#       the program will now determine if the isRunning file has been around
#       too long. If so, the old time of the previous instance is overwritten.
#       Also, in the event that the isRunning file is overwritten, there will
#       be an output statement printed to the cron daemon to inform operations
#       monitoring there may be an issue with the runtime job.  
###############################################################################
if( -e $thisProgIsRunningFile ) {
  $priorRunTime = `/bin/cat $thisProgIsRunningFile`;
  $secondsDiff = $theCurrentTime - $priorRunTime;
  if( $secondsDiff < $maxTime2Run ){
#    print "Difference in seconds since last run = $secondsDiff \n";
#    print "Exiting this instance since another is running...\n";
    exit(0);
  } else {
    print "$thisProgIsRunningFile has been around more than $maxTime2Run.\n";
    print "Removing the isRunning check file and starting next iteration.\n";
    system( "/bin/echo $theCurrentTime > $thisProgIsRunningFile" );
  }
} else {
  system( "/bin/echo $theCurrentTime > $thisProgIsRunningFile" );
}


#&Date_Init(); # This must be done to guarantee the DateManip routines
#              # are ready for use.

##GLOBAL VARIABLES#############################################################
local $logFPN = $dataDir . "/ecf.log";
#local $logFPN = "$ARGV[0]";

##END GLOBAL VARIABLES#########################################################

##SUBROUTINES##################################################################

sub getBaseName {
  $inString = shift;
  $inString =~ /(\/\w+)*\/(.*)$/;
  return $2;
}

##END SUBROUTINES##############################################################

($currSecs,$currMin,$currHr,$currDay,$currMon,$currYr,$currWDay,$currJDay,$currISDST) = gmtime( time() );
$currGM_ISO_Date = sprintf( "%4d%02d%02d", $currYr+1900, $currMon+1, $currDay );

print "DEBUG:currGM_ISO_Date=$currGM_ISO_Date, currDay=$currDay\n";
print "DEBUG: currWDay=$currWDay, currJDay=$currJDay, currISDST=$currISDST\n";
$previousStateDir = "$interProcDir/";
$previousStateFPN_sub = $previousStateDir . "prev_ECF_Submitted.txt";
$previousStateFPN_act = $previousStateDir . "prev_ECF_Active.txt";
$previousStateFPN_limbo = $previousStateDir . "limbo.hold";
$completeBeforeActiveFPN = $previousStateDir . $currGM_ISO_Date . ".compB4Active";
#$fullyCompleteJobTimingFPN = $previousStateDir . "../" . $currGM_ISO_Date . ".log";
$rejectedProcessesFPN = $previousStateDir . $currGM_ISO_Date . ".dropped";
$prevRunLineNumberFile = $previousStateDir . "prevRunECFLineNumber.txt";

# Get records from submitted state file
if( -e $previousStateFPN_sub ) {
  open( $IN, "< $previousStateFPN_sub" ) or die "Can't open $previousStateFPN_sub for reading.:$!\n";
  # fill input hash with record of submitted or active job that hasn't
  # run to completion before the last line of the ecf.log file
  # was processed.
  while( $record = <$IN> ){
    chomp( $record );
    if( $record =~ /<N>(.+?)<\/N><S>(\d+)<\/S>/ ){
      $jobName = $1;
      $ss1970 = $2;
      if( ($currGM_ISO_Date - $ss1970) < 36000 ) {
        $submittedJobs{ $jobName }[0] = $ss1970;
      }
    }
  }
  close( $IN );
  unlink( $previousStateFPN_sub );
} else {
  $submittedJobs{ "" }[0] = 0;
}

# Get records from active state file
if( -e $previousStateFPN_act ) {
  open( $IN, "< $previousStateFPN_act" ) or die "Can't open $previousStateFPN_act for reading.:$!\n";
  # fill input hash with record of submitted or active job that hasn't
  # run to completion before the last line of the ecf.log file
  # was processed.
  while( $record = <$IN> ){
    chomp( $record );
    if( $record =~ /<N>(.+?)<\/N><A>(\d+)<\/A>/ ){
      $jobName = $1;
      $ss1970 = $2;
      if( ($currGM_ISO_Date - $ss1970) < 36000 ) {
        $activeJobs{ $jobName }[0] = $ss1970;
      }
    }
  }
  close( $IN );
  unlink( $previousStateFPN_act );
} else {
  $activeJobs{ "" }[0] = 0;
}

# Get line count from previous run of ecf.log processing

if( -e $prevRunLineNumberFile ){
  $prevRunLineNumber = `/bin/cat $prevRunLineNumberFile`;
  chomp( $prevRunLineNumber );
  $prevRunLineNumber;
} else {
  $prevRunLineNumber = 0;
}

# Get the contents of the current ecf.log and place into a scalar list.
# I chose to grab the entire available log file content
# since I was concerned that other methods might cause a line
# or two to be skipped. If the last line isn't complete then
# decrement the logLineCount variable so that line is reprocessed
# during the next interation of this program.

$ecfLogLinesList = `/bin/cat $logFPN`;
@ecfLogLinesArr = split( /^/, $ecfLogLinesList );
undef( $ecfLogLinesList );
$logLineCount = $#ecfLogLinesArr;
$origLogLineCount = 0;
######################################################################
# This section attempts to capture the content of the rolled out log file
# that was moved due to zulu rollover.
######################################################################
print "DEBUG:prevRunLineNumber=$prevRunLineNumber, logLineCount=$logLineCount, notMachineChanged=$notMachineChanged.\n";
if( ( $prevRunLineNumber > $logLineCount ) & $notMachineChanged ){
  $currentLineIndex = 0;
  # JY $copyString = "-q ecfprod\@$theECFMachine:/ecf/rundir/log/ecf.$theECFMachine.log1 $dataDir/ecflog1 2>&1 > /dev/null";
  # system( "/usr/bin/scp $copyString" );
  $copyString = "-p $COMIN_ECFLOG/log/ecf.$theECFMachine.log1 $dataDir/ecflog1 2>&1 > /dev/null";
  print "DEBUG2:copyString=$copyString, theECFMachine=$theECFMachine \n";
  system( "/usr/bin/cp $copyString" );
  $log1LineCount = `wc -l $dataDir/ecflog1`;
  chomp( $log1LineCount );
  $countDiff = $log1LineCount - $prevRunLineNumber;
  $trailingLogList = `/usr/bin/tail \-$countDiff $dataDir/ecflog1`;
  chomp( $trailingLogList );
  @finalLinesArr = split( /^/, $trailingLogList );
  $tempArrSize1 = $#finalLinesArr;
  push( @finalLinesArr, @ecfLogLinesArr );
  undef( @ecfLogLinesArr );
  @ecfLogLinesArr = @finalLinesArr;
  $tempArrSize2 = $#ecfLogLinesArr;
  $origLogLineCount = $logLineCount;
  $logLineCount = $#ecfLogLinesArr;
} elsif( $prevRunLineNumber == 0 ) {
  $currentLineIndex = 0;
} else {
  if( $logLineCount != $prevRunLineNumber ){
    $currentLineIndex = $prevRunLineNumber + 1;
  } else {
    $currentLineIndex = $prevRunLineNumber;
  }
}

###############################################################################
# The activity for any job must follow the specific sequence of events
# listed in the following sample lines. But note that there may be
# other job entries in the ecf.log file occurring between the entries
# for the job in which you are interested. All lines for other jobs
# have been removed to make the sequence easier to read and understand.
#
# LOG:[23:55:54 15.3.2004] submitted:/prod12/fnlibm18/jfnl18_dump_alert
# LOG:[23:55:59 15.3.2004] active:/prod12/fnlibm18/jfnl18_dump_alert
# LOG:[23:56:00 15.3.2004] complete:/prod12/fnlibm18/jfnl18_dump_alert
#
# If there is some event that removes a job from the active state,
# prior to being marked complete, the job will be removed from the
# job submitted table.
###############################################################################
###############################################################################
# Sample Output:
#/prod12/sfcrucibm12/rsasibm12_HH22/jsruc_dump_23   1.1.2005 23:22:53  1.1.2005 23:23:45
#/prod12/hireswibm18/nmm_ctl   1.1.2005 23:18:49  1.1.2005 23:25:27
#/prod12/sfcrucibm12/rsasibm12_HH22_23   1.1.2005 23:26:07  1.1.2005 23:26:15
#/prod12/wavibm18   1.1.2005 22:47:44  1.1.2005 23:26:38
#/prod12/rucibm18/jruc2a_dump_23   1.1.2005 23:24:52  1.1.2005 23:28:38
#/prod12/sfcrucibm12   1.1.2005 23:26:07  1.1.2005 23:26:15
#/prod12/rucibm18/jruc2a_dump_post_23   1.1.2005 23:28:44  1.1.2005 23:29:50
###############################################################################
# Skip to the next line in the log file that hasn't been processed
$lastLineIsNotComplete = 0;
open( $COMP_B4_ACTIVE, ">>$completeBeforeActiveFPN" ) or die "Can't open file ($completeBeforeActiveFPN) for writing.:$!\n";
open( $REJECTED_JOBS, ">>$rejectedProcessesFPN" ) or die "Can't open file ($rejectedProcessesFPN) for writing.:$!\n";

for( $i = $currentLineIndex; $i <= $logLineCount; $i++ ){

  $logLine = $ecfLogLinesArr[ $i ];
  chomp( $logLine );

  if( $logLine =~ /\w{3}:\[(\d\d):(\d\d):(\d\d)\s+(\d{1,2})\.(\d{1,2})\.(\d{4})\]\s+(.+)/ ){
    @logLineSplit = split(/:/ , $logLine);
    $msgType = $logLineSplit[0];
    $jhr = $1;
    $jmin = $2;
    $jsec = $3;
    $jday = $4;
    $jmon = $5;
    $jyr  = $6;
    $operation = $7;
#    if( $operation =~ /(.*?)\s+(.*)$/ ){
#       $operation = $1;
#    }
    chomp( $operation );
  } else {
    if( $i == $logLineCount ) {
      # Do this since the line is not complete. Process the next interation. 
      $lastLineIsNotComplete = 1;
#      print "Incomplete line.\n";
    } else {
#      print "Line format bad: ($i) ($logLine) ($logLineCount).\n";
    }
    next;
  }

  ($act1,$jobPN) = split( /:\s/, $operation );
  ($jobPN,$junk) = split( /\s/, $jobPN );
# $dateTimeInSeconds = Date_SecsSince1970( $jmon, $jday, $jyr, $jhr, $jmin, $jsec );
  $dateTimeInSeconds = Time::Piece->strptime("$jmon $jday $jyr $jhr $jmin $jsec","%m %d %Y %H %M %S")->epoch;

  if( $msgType eq "LOG" & $act1 =~ /^submitted/ ){
    $submittedJobs{$jobPN}[0] = $dateTimeInSeconds;
  } elsif( $msgType eq "LOG" & $act1 =~ /^active/ ){
    ##################################################################
    # Don't process any records with an active time that is older than 
    #  one day from current day
    ##################################################################
    if( ( $theCurrentTime - $dateTimeInSeconds ) < 86400 ){
      $activeJobs{$jobPN}[0] = $dateTimeInSeconds;
    } else {
      delete( $submittedJobs{$jobPN}[0] );
    }
  } elsif( $msgType eq "LOG" & $act1 =~ /^complete/ ){
    # Don't do anything with this record if no start time was established
    if( ($activeJobs{$jobPN}[0] == 0) && ($submittedJobs{$jobPN}[0] == 0) ){
      print $COMP_B4_ACTIVE "<N>$jobPN</N><C>$dateTimeInSeconds</C><L>$i</L>\n";
    } else {
      $tempKey = $activeJobs{$jobPN}[0] . "_-_" . $jobPN;
      $completedJobs{$tempKey}[0] = $submittedJobs{$jobPN}[0];
      $completedJobs{$tempKey}[1] = $activeJobs{$jobPN}[0];
      $completedJobs{$tempKey}[2] = $dateTimeInSeconds;
      delete( $submittedJobs{$jobPN}[0] );
      delete( $activeJobs{$jobPN}[0] );
    }
  } elsif( $msgType eq "WAR" & $act1 =~ /^abort/ ){
    if( exists( $submittedJobs{$jobPN} ) || exists( $activeJobs{$jobPN} ) ){
      $subTime = $submittedJobs{$jobPN}[0];
      $actTime = $activeJobs{$jobPN}[0];
      print $REJECTED_JOBS "<N>$jobPN</N><I>abort</I><S>$subTime</S><A>$actTime</A><R>$dateTimeInSeconds</R><L>$i</L>\n";
      delete( $submittedJobs{$jobPN} );
      delete( $activeJobs{$jobPN} );
    }
  } elsif( $msgType eq "LOG" & $act1 =~ /^aborted/ ){
    if( exists( $submittedJobs{$jobPN} ) || exists( $activeJobs{$jobPN} ) ){
      $subTime = $submittedJobs{$jobPN}[0];
      $actTime = $activeJobs{$jobPN}[0];
      print $REJECTED_JOBS "<N>$jobPN</N><I>aborted</I><S>$subTime</S><A>$actTime</A><R>$dateTimeInSeconds</R><L>$i</L>\n";
      delete( $submittedJobs{$jobPN} );
      delete( $activeJobs{$jobPN} );
    }
  } elsif( $msgType eq "MSG" & $act1 =~ /^force/ ){
    if( exists( $submittedJobs{$jobPN} ) || exists( $activeJobs{$jobPN} ) ){
      $subTime = $submittedJobs{$jobPN}[0];
      $actTime = $activeJobs{$jobPN}[0];
      print $REJECTED_JOBS "<N>$jobPN</N><I>force</I><S>$subTime</S><A>$actTime</A><R>$dateTimeInSeconds</R><L>$i</L>\n";
      delete( $submittedJobs{$jobPN} );
      delete( $activeJobs{$jobPN} );
    }
  } elsif( $msgType eq "MSG" & $act1 =~ /^suspend/ ){
    if( exists( $submittedJobs{$jobPN} ) || exists( $activeJobs{$jobPN} ) ){
      $subTime = $submittedJobs{$jobPN}[0];
      $actTime = $activeJobs{$jobPN}[0];
      print $REJECTED_JOBS "<N>$jobPN</N><I>suspend</I><S>$subTime</S><A>$actTime</A><R>$dateTimeInSeconds</R><L>$i</L>\n";
      delete( $submittedJobs{$jobPN} );
      delete( $activeJobs{$jobPN} );
    }
  }

  $currentLineIndex = $i;
}

close( $REJECTED_JOBS );
system( "/bin/chmod 775 $rejectedProcessesFPN" );
close( $COMP_B4_ACTIVE );
system( "/bin/chmod 775 $completeBeforeActiveFPN" );

if( $lastLineIsNotComplete ){
  # Must subtract 2 to compensate for the increment in previous loop and bad
  # format of final line in the log file.
  $currentLineIndex -= 2;
}

open( STATS, "$ave30dfull" );
while ( <STATS> ) {
    chop;
    ( $jid, $avestart, $avestop, $a1, $b1, $a2, $b2, $a3, $b3, $a4, $b4,
     $a5, $b5, $a6, $b6, $a7, $b7, $a8, $b8, $a9, $b9, $a10, $b10,
     $a11, $b11, $a12, $b12, $a13, $b13, $a14, $b14, $a15, $b15, $a16, $b16,
     $a17, $b17, $a18, $b18, $a19, $b19, $a20, $b20, $a21, $b21, $a22, $b22,
     $a23, $b23, $a24, $b24, $a25, $b25, $a26, $b26, $a27, $b27, $a28, $b28,
     $a29, $b29, $a30, $b30 ) = split( /\s+/ );

    $job30{$jid} = $jid;
    $a30start{$jid} = $avestart;
    $a30stop{$jid} = $avestop;
    $start30{$jid,1} = $a1;
    $stop30{$jid,1} = $b1;
    $start30{$jid,2} = $a2;
    $stop30{$jid,2} = $b2;
    $start30{$jid,3} = $a3;
    $stop30{$jid,3} = $b3;
    $start30{$jid,4} = $a4;
    $stop30{$jid,4} = $b4;
    $start30{$jid,5} = $a5;
    $stop30{$jid,5} = $b5;
    $start30{$jid,6} = $a6;
    $stop30{$jid,6} = $b6;
    $start30{$jid,7} = $a7;
    $stop30{$jid,7} = $b7;
    $start30{$jid,8} = $a8;
    $stop30{$jid,8} = $b8;
    $start30{$jid,9} = $a9;
    $stop30{$jid,9} = $b9;
    $start30{$jid,10} = $a10;
    $stop30{$jid,10} = $b10;
    $start30{$jid,11} = $a11;
    $stop30{$jid,11} = $b11;
    $start30{$jid,12} = $a12;
    $stop30{$jid,12} = $b12;
    $start30{$jid,13} = $a13;
    $stop30{$jid,13} = $b13;
    $start30{$jid,14} = $a14;
    $stop30{$jid,14} = $b14;
    $start30{$jid,15} = $a15;
    $stop30{$jid,15} = $b15;
    $start30{$jid,16} = $a16;
    $stop30{$jid,16} = $b16;
    $start30{$jid,17} = $a17;
    $stop30{$jid,17} = $b17;
    $start30{$jid,18} = $a18;
    $stop30{$jid,18} = $b18;
    $start30{$jid,19} = $a19;
    $stop30{$jid,19} = $b19;
    $start30{$jid,20} = $a20;
    $stop30{$jid,20} = $b20;
    $start30{$jid,21} = $a21;
    $stop30{$jid,21} = $b21;
    $start30{$jid,22} = $a22;
    $stop30{$jid,22} = $b22;
    $start30{$jid,23} = $a23;
    $stop30{$jid,23} = $b23;
    $start30{$jid,24} = $a24;
    $stop30{$jid,24} = $b24;
    $start30{$jid,25} = $a25;
    $stop30{$jid,25} = $b25;
    $start30{$jid,26} = $a26;
    $stop30{$jid,26} = $b26;
    $start30{$jid,27} = $a27;
    $stop30{$jid,27} = $b27;
    $start30{$jid,28} = $a28;
    $stop30{$jid,28} = $b28;
    $start30{$jid,29} = $a29;
    $stop30{$jid,29} = $b29;
    $start30{$jid,30} = $a30;
    $stop30{$jid,30} = $b30;


}
close STATS;

###############################################################################
# Output processed data
###############################################################################

$fullyCompleteJobTimingFPN = $COMOUTruntime . $currGM_ISO_Date . ".log";
$fullyCompleteJobTimingFPN_new = $COMOUTdaily . $currGM_ISO_Date . ".daily";
$fullyCompleteJobTimingFPN2 = $COMOUTruntime2 . $currGM_ISO_Date . ".log";
open( $OUT, ">>$fullyCompleteJobTimingFPN" ) or die "Can't open $fullyCompleteJobTimingFPN for output.:!$\n";
open( $OUTNEW, ">>$fullyCompleteJobTimingFPN_new" ) or die "Can't open $fullyCompleteJobTimingFPN_new for output.:!$\n";
open( $OUT2, ">>$fullyCompleteJobTimingFPN2" ) or die "Can't open $fullyCompleteJobTimingFPN2 for output.:!$\n";

foreach $completedJob ( sort keys( %completedJobs ) ){

  ( $secsKP, $theJobNameKP ) = split( /_-_/, $completedJob, 2 );
  $submitSecs = $completedJobs{ $completedJob }[0];
  $activeSecs = $completedJobs{ $completedJob }[1];
  $completeSecs = $completedJobs{ $completedJob }[2];

  if( $submitSecs > 0 && $activeSecs > 0 ) {
    $sub2Act_Seconds  = ( $activeSecs - $submitSecs );
    $act2Comp_Seconds = ( $completeSecs - $activeSecs );
  } elsif( $submitSecs <= 0 && $activeSecs > 0 ) {
    $sub2Act_Seconds = 0;
    $act2Comp_Seconds = ( $completeSecs - $activeSecs );
  } elsif( $submitSecs > 0 && $activeSecs <= 0 ) {
    $sub2Act_Seconds = 0;
    $act2Comp_Seconds = ( $completeSecs - $submitSecs );
  } else {
    $sub2Act_Seconds = 0;
    $act2Comp_Seconds = 0;
  }

  $byr  = (gmtime($submitSecs))[5] + 1900;
  $bmon = (gmtime($submitSecs))[4] + 1;
  $bday = (gmtime($submitSecs))[3];
  $bhr  = (gmtime($submitSecs))[2];
  $bmin = (gmtime($submitSecs))[1];
  $bsec = (gmtime($submitSecs))[0];

  $syr  = (gmtime($activeSecs))[5] + 1900;
  $smon = (gmtime($activeSecs))[4] + 1;
  $sday = (gmtime($activeSecs))[3];
  $shr  = (gmtime($activeSecs))[2];
  $smin = (gmtime($activeSecs))[1];
  $ssec = (gmtime($activeSecs))[0];

  $eyr  = (gmtime($completeSecs))[5] + 1900;
  $emon = (gmtime($completeSecs))[4] + 1;
  $eday = (gmtime($completeSecs))[3];
  $ehr  = (gmtime($completeSecs))[2];
  $emin = (gmtime($completeSecs))[1];
  $esec = (gmtime($completeSecs))[0];

#  printf $OUT "%s   %02d.%02d.%04d %02d:%02d:%02d  %02d.%02d.%04d %02d:%02d:%02d  %02d.%02d.%04d %02d:%02d:%02d\n", $theJobNameKP,$bday,$bmon,$byr,$bhr,$bmin,$bsec,$sday,$smon,$syr,$shr,$smin,$ssec,$eday,$emon,$eyr,$ehr,$emin,$esec;
#  printf $OUT "<N>%s</N><S>%04d%02d%02d%02d%02d%02d</S><A>%04d%02d%02d%02d%02d%02d</A><C>%04d%02d%02d%02d%02d%02d</C>\n", $theJobNameKP,$byr,$bmon,$bday,$bhr,$bmin,$bsec,$syr,$smon,$sday,$shr,$smin,$ssec,$eyr,$emon,$eday,$ehr,$emin,$esec;
  if( $sub2Act_Seconds >= 0 & $act2Comp_Seconds >= 0 ){
    #printf $OUT "%-72s%04d%02d%02d%02d%02d %04d%02d%02d%02d%02d %03d %3.1f\n", $theJobNameKP,$syr,$smon,$sday,$shr,$smin,$eyr,$emon,$eday,$ehr,$emin,$sub2Act_Seconds,$act2Comp_Seconds/60;
    printf $OUT "%s   %d.%d.%04d %02s:%02s:%02s  %d.%d.%04d %02s:%02s:%02s\n", $theJobNameKP,$sday,$smon,$syr,$shr,$smin,$ssec,$eday,$emon,$eyr,$ehr,$emin,$esec;
    printf $OUTNEW "%-30s %04d%02d%02d %02d:%02d %04d%02d%02d %02d:%02d %4d %6.1f\n", $theJobNameKP,$syr,$smon,$sday,$shr,$smin,$eyr,$emon,$eday,$ehr,$emin,$sub2Act_Seconds,$act2Comp_Seconds/60;
    printf $OUT2 "<N>%s</N><S>%04d%02d%02d%02d%02d%02d</S><A>%04d%02d%02d%02d%02d%02d</A><C>%04d%02d%02d%02d%02d%02d</C><SA>%d</SA><AC>%d</AC>\n", $theJobNameKP,$byr,$bmon,$bday,$bhr,$bmin,$bsec,$syr,$smon,$sday,$shr,$smin,$ssec,$eyr,$emon,$eday,$ehr,$emin,$esec,$sub2Act_Seconds,$act2Comp_Seconds;
  }
  $starttime=$shr * 60 + $smin + $ssec / 60;
  $stoptime=$ehr * 60 + $emin + $esec / 60;
  if ( $a30start{$theJobNameKP} ne "" ) {
       $new30start{$theJobNameKP} = $starttime;
       $new30stop{$theJobNameKP} = $stoptime;
       if ( $a30start{$theJobNameKP} < 360 && $new30start{$theJobNameKP} > 1080 ) {
           $new30start{$theJobNameKP} = $new30start{$theJobNameKP} - 1440;
       }
       if ( $a30stop{$theJobNameKP} < 360 && $new30stop{$theJobNameKP} > 1080 ) {
           $new30stop{$theJobNameKP} = $new30stop{$theJobNameKP} - 1440;
       }
       if ( $a30start{$theJobNameKP} > 1080 && $new30start{$theJobNameKP} < 360 ) {
           $new30start{$theJobNameKP} = $new30start{$theJobNameKP} + 1440;
       }
       if ( $a30stop{$theJobNameKP} > 1080 && $new30stop{$theJobNameKP} < 360 ) {
           $new30stop{$theJobNameKP} = $new30stop{$theJobNameKP} + 1440;
       }
       $tempavestart = $new30start{$theJobNameKP};
       $tempavestop = $new30stop{$theJobNameKP};
       for ( $i = 1; $i <= 29; $i++ ) {
                  if ( $start30{$theJobNameKP,$i} eq "" || $start30{$theJobNameKP,$i} eq " " ) {
               $start30{$theJobNameKP,$i} = $new30start{$theJobNameKP};
           }
           if ( $stop30{$theJobNameKP,$i} eq "" || $stop30{$theJobNameKP,$i} eq " " ) {
               $stop30{$theJobNameKP,$i} = $new30stop{$theJobNameKP};
           }
           $tempavestart = $tempavestart + $start30{$theJobNameKP,$i};
           $tempavestop = $tempavestop + $stop30{$theJobNameKP,$i};
       }
       $a30start{$theJobNameKP} = $tempavestart / 30;
       $a30stop{$theJobNameKP} = $tempavestop / 30;
   } else {
       #
       # This is the first time this job has
       # appeared.  Set the averages and 30 day
       # history to the initial value 
       #
       $job30{$theJobNameKP} = $theJobNameKP;
       $a30start{$theJobNameKP} = $starttime;
       $a30stop{$theJobNameKP} = $stoptime;
       $new30start{$theJobNameKP} = $starttime;
       $new30stop{$theJobNameKP} = $stoptime;
       for ( $i = 1; $i <= 29; $i++ ) {
           $start30{$theJobNameKP,$i} = $starttime;
           $stop30{$theJobNameKP,$i} = $stoptime;
       }
   }

}
close( $OUT2 );
undef( $OUT2 );
close( $OUTNEW );
undef( $OUTNEW );
close( $OUT );
undef( $OUT );
system( "/bin/chmod 775 $fullyCompleteJobTimingFPN" );
system( "/bin/chmod 775 $fullyCompleteJobTimingFPN_new" );
system( "/bin/chmod 775 $fullyCompleteJobTimingFPN2" );

open( $OUT, "> $previousStateFPN_sub" ) or die "Can't open ($previousStateFPN_sub) for writing.:$!\n";
foreach $submittedJob ( sort keys( %submittedJobs ) ){
  $dtInSecs = $submittedJobs{$submittedJob}[0];
  if( $dtInSecs != 0 ){
    printf $OUT "<N>%s</N><S>%ld</S>\n", $submittedJob, $dtInSecs;
  }
}
close( $OUT );
undef( $OUT );
system( "/bin/chmod 775 $previousStateFPN_sub" );

open( $OUT, "> $previousStateFPN_act" ) or die "Can't open ($previousStateFPN_act) for writing.:$!\n";
foreach $activeJob ( sort keys( %activeJobs ) ){
  $dtInSecs = $activeJobs{$activeJob}[0];
  if( $dtInSecs != 0 ){
    printf $OUT "<N>%s</N><A>%ld</A>\n", $activeJob, $dtInSecs;
  }
}
close( $OUT );
undef( $OUT );
system( "/bin/chmod 775 $previousStateFPN_act" );

open( $OUT, "> $previousStateFPN_limbo" ) or die "Can't open ($previousStateFPN_limbo) for writing.:$!\n";
foreach $activeJob ( sort keys( %activeJobs ) ){
  $dtInSecs = $activeJobs{$activeJob}[0];
  if( $dtInSecs != 0 ){
#   ($limbo_yyyymmddhh,$limbo_min,$limbo_ss) = split( /:/, ParseDateString("epoch $dtInSecs"));
    ($limbo_yyyymmddhh,$limbo_min,$limbo_ss) = split( /:/, Time::Piece->strptime($dtInSecs,"%s")->strftime("%Y%m%d%H:%M:%S"));
    $limbo_yyyy=substr $limbo_yyyymmddhh, 0, -6; 
    $limbo_mon=substr $limbo_yyyymmddhh, 4, -4;
    $limbo_dd=substr $limbo_yyyymmddhh, 6, -2;
    $limbo_hh=substr $limbo_yyyymmddhh, 8;
    printf $OUT "%s %d.%d.%d %01s:%01s:%01s\n", $activeJob, $limbo_dd, $limbo_mon, $limbo_yyyy, $limbo_hh, $limbo_min, $limbo_ss;
  }
}
close( $OUT );
undef( $OUT );
system( "/bin/chmod 775 $previousStateFPN_limbo" );

# Used a little trick here by normally setting $origLogLineCount to
# zero and assigning a value only at the time when a new log file
# has been encountered.

if( ! $origLogLineCount ){
  system( "/bin/echo $currentLineIndex > $prevRunLineNumberFile" );
} else {
  system( "/bin/echo $origLogLineCount > $prevRunLineNumberFile" );
}
system( "/bin/chmod 775 $prevRunLineNumberFile" );


##########################################################
# This is an add-on section; taken from /nwprod/ush/runstats.pl
# Writes out 30-day average
# Output files: ave30day.short; ave30day.full
##########################################################

#
# Write out 30-day average files
#
open( AVELONG, ">$ave30dfull" );
open( AVESHORT, ">$ave30day" );

@keys = keys %job30;
while (@keys) {
    $temp=pop(@keys);
    select ( AVELONG );
    if ( $new30start{$temp} ne "" ) {
	print "$temp $a30start{$temp} $a30stop{$temp} $new30start{$temp} $new30stop{$temp}";
	for ( $i =1; $i <= 29; $i++ ) {
	    print " $start30{$temp,$i} $stop30{$temp,$i}";
	}
    } else {
	print "$temp $a30start{$temp} $a30stop{$temp}";
	for ( $i =1; $i <= 30; $i++ ) {
            print " $start30{$temp,$i} $stop30{$temp,$i}";
        }
    }
    print "\n";
    select ( AVESHORT );
    print $temp,' ',$a30start{$temp},' ',$a30stop{$temp},' ',$a30stop{$temp}-$a30start{$temp},"\n";
}
close AVELONG;
close AVESHORT;


unlink( $thisProgIsRunningFile );
