#!/usr/bin/perl
# This script accepts a transfer template file where each package is denoted by a
# line beginning with "nwprod/" followed by the package name.  The latest version
# of each package mentioned in the template is found and those packages that are
# located in the current $NWROOT are written to the transfer list file with the
# location of the latest version replacing the "nwprod/${package}" specifier.
# If the $RUN environment variable is "nwprod_cleanup", the output file will contain
# a list file that excludes the latest versions of the packages in the template.
#
# Usage:
#   export RUN=nwprod_xc40
#   export NWROOT=/gpfs/hps/nco/ops/nwprod
#   get_package_versions.pl $TRANSFER_TEMPLATE $TRANSFER_LIST
#
# History:
#   2016-12-14  Kit Menlove  Initial script
#   2017-01-03  Kit Menlove  Updated to use version.pm instead of CPAN/Version.pm
#   2017-01-10  Kit Menlove  Updated to handle decoders and NCEP production libraries
#   2020-04-22  Jianbin Yang Updated to ignore the '-' in the version number

use strict;
use version;
use Cwd;

my $error_messages = "";
$error_messages .= "Please give the location of the (input) transfer template file.\n" if (@ARGV < 1);
$error_messages .= "Please give the location of the (output) transfer list file.\n" if (@ARGV < 2);
$error_messages .= "Please set the RUN environment variable.\n" unless (exists $ENV{'RUN'});
$error_messages .= "Please set the NWROOT environment variable.\n" unless (exists $ENV{'NWROOT'});
die $error_messages if ($error_messages);

my $transfer_template = shift @ARGV;
my $transfer_list = shift @ARGV;
my $save_bad_lines_in_version_files = 0;

my %package_info=();

# Decoders directory location
if (exists $ENV{'NWROOTdecoders'}) {
    $package_info{'decoders'} = ["", $ENV{'NWROOTdecoders'}];
} else {
    print STDERR "WARNING: The \$NWROOTdecoders environment variable is not set!\n";
}

# Lib directory location
if (exists $ENV{'NWROOTlib'}) {
    $package_info{'lib'} = ["", $ENV{'NWROOTlib'}];
} else {
    print STDERR "WARNING: The \$NWROOTlib environment variable is not set!\n";
}

open(BAD_LINES, ">bad_lines.txt") if ($save_bad_lines_in_version_files);

# Look through all version files and find the highest version number for each package cited
# together with its location.  Store this information in the package_info hash table.
#while (my $filename = <\/gpfs\/?p?\/nco\/ops\/nwprod\/versions\/*.ver>) {
while (my $filename = <\/gpfs\/hps\/nco\/ops\/nwprod\/versions\/*.ver>) {
    $filename =~ /(\/gpfs\/\w{3}\/nco\/ops\/nwprod)\/versions\/(\w+)\.ver/;
    my $version_file_nwroot = $1;
    my $version_file_name = $2;
    open(VERSION_FILE, $filename);
    while (<VERSION_FILE>) {
      #  if (/(\w+)_ver=v?(\d+\.\S*)/) {
      #  if (/(\w+)_ver=\"?v?(\d+\.[^\"\s]*)/) {
        if (/(\w+)_ver=\"?v?(\d+\.[^\"\s\-]*)/) {
            my $package = $1;
            my $version = "v$2";
            if ($package eq "model") {
                $package = $version_file_name;
            }
            # Make sure the directory exists in the same phase as the version file
            # before we assume we have a valid version
            if (! -d "$version_file_nwroot/$package.$version") {
                # If the package doesn't exist, see if it's on a different phase
                my @potential_dirnames = <\/gpfs\/?p?\/nco\/ops\/nwprod\/$package.$version>;
                if (@potential_dirnames) {
                    if (@potential_dirnames > 1) {
                        print STDERR "WARNING: Found more than one match for this package: " . (join " and ", @potential_dirnames) . "\n";
                    }
                    $potential_dirnames[0] =~ /(\/gpfs\/\w{3}\/nco\/ops\/nwprod)\/$package.$version/;
                    $version_file_nwroot = $1;
                } else {
                    print STDERR "WARNING: $package.$version reported in $filename does not exist!\n";
                    next;
                }
            }
            if (exists $package_info{$package}) {
                print STDERR "CHECKING: $package_info{$package}[1] $package $version $package_info{$package}[0] !!!!\n";
                if (version->new($version) > version->new($package_info{$package}[0])) {
                    $package_info{$package}[0] = $version;
                    $package_info{$package}[1] = $version_file_nwroot;
                }
            } else {
                $package_info{$package} = [$version, $version_file_nwroot];
            }
        } elsif ($save_bad_lines_in_version_files) {
            print BAD_LINES;
        }
    }
    close VERSION_FILE;
}

while (my $filename = <\/gpfs\/dell1\/nco\/ops\/nwprod\/versions\/*.ver>) {
    $filename =~ /(\/gpfs\/\w{5}\/nco\/ops\/nwprod)\/versions\/(\w+)\.ver/;
    my $version_file_nwroot = $1;
    my $version_file_name = $2;
    open(VERSION_FILE, $filename);
    while (<VERSION_FILE>) {
      #  if (/(\w+)_ver=v?(\d+\.\S*)/) {
      #  if (/(\w+)_ver=\"?v?(\d+\.[^\"\s]*)/) {
        if (/(\w+)_ver=\"?v?(\d+\.[^\"\s\-]*)/) {
            my $package = $1;
            my $version = "v$2";
            if ($package eq "model") {
                $package = $version_file_name;
            }
            # Make sure the directory exists in the same phase as the version file
            # before we assume we have a valid version
            if (! -d "$version_file_nwroot/$package.$version") {
                # If the package doesn't exist, see if it's on a different phase
                my @potential_dirnames = <\/gpfs\/dell1\/nco\/ops\/nwprod\/$package.$version>;
                if (@potential_dirnames) {
                    if (@potential_dirnames > 1) {
                        print STDERR "WARNING: Found more than one match for this package: " . (join " and ", @potential_dirnames) . "\n";
                    }
                    $potential_dirnames[0] =~ /(\/gpfs\/\w{5}\/nco\/ops\/nwprod)\/$package.$version/;
                    $version_file_nwroot = $1;
                } else {
                    print STDERR "WARNING: $package.$version reported in $filename does not exist!\n";
                    next;
                }
            }
            if (exists $package_info{$package}) {
                print STDERR "CHECKING: $package_info{$package}[1] $package $version $package_info{$package}[0] !!!!\n";
                if (version->new($version) > version->new($package_info{$package}[0])) {
                    $package_info{$package}[0] = $version;
                    $package_info{$package}[1] = $version_file_nwroot;
                }
            } else {
                $package_info{$package} = [$version, $version_file_nwroot];
            }
        } elsif ($save_bad_lines_in_version_files) {
            print BAD_LINES;
        }
    }
    close VERSION_FILE;
}

while (my $filename = <\/gpfs\/dell4\/nco\/ops\/nwprod\/versions\/*.ver>) {
    $filename =~ /(\/gpfs\/\w{5}\/nco\/ops\/nwprod)\/versions\/(\w+)\.ver/;
    my $version_file_nwroot = $1;
    my $version_file_name = $2;
    open(VERSION_FILE, $filename);
    while (<VERSION_FILE>) {
      #  if (/(\w+)_ver=v?(\d+\.\S*)/) {
      #  if (/(\w+)_ver=\"?v?(\d+\.[^\"\s]*)/) {
        if (/(\w+)_ver=\"?v?(\d+\.[^\"\s\-]*)/) {
            my $package = $1;
            my $version = "v$2";
            if ($package eq "model") {
                $package = $version_file_name;
            }
            # Make sure the directory exists in the same phase as the version file
            # before we assume we have a valid version
            if (! -d "$version_file_nwroot/$package.$version") {
                # If the package doesn't exist, see if it's on a different phase
                my @potential_dirnames = <\/gpfs\/dell4\/nco\/ops\/nwprod\/$package.$version>;
                if (@potential_dirnames) {
                    if (@potential_dirnames > 1) {
                        print STDERR "WARNING: Found more than one match for this package: " . (join " and ", @potential_dirnames) . "\n";
                    }
                    $potential_dirnames[0] =~ /(\/gpfs\/\w{5}\/nco\/ops\/nwprod)\/$package.$version/;
                    $version_file_nwroot = $1;
                } else {
                    print STDERR "WARNING: $package.$version reported in $filename does not exist!\n";
                    next;
                }
            }
            if (exists $package_info{$package}) {
                print STDERR "CHECKING: $package_info{$package}[1] $package $version $package_info{$package}[0] !!!!\n";
                if (version->new($version) > version->new($package_info{$package}[0])) {
                    $package_info{$package}[0] = $version;
                    $package_info{$package}[1] = $version_file_nwroot;
                }
            } else {
                $package_info{$package} = [$version, $version_file_nwroot];
            }
        } elsif ($save_bad_lines_in_version_files) {
            print BAD_LINES;
        }
    }
    close VERSION_FILE;
}

close BAD_LINES if ($save_bad_lines_in_version_files);

# Special case for the prod_util package which is loaded automatically and not in any
# version files
open(VERSION_FILE, "/gpfs/dell1/nco/ops/nwprod/modulefiles/prod_util/.version");
while (<VERSION_FILE>) {
    if (/set ModulesVersion\s+"(\d+\.\S+)"/i) {
        $package_info{'prod_util'} = [$1, "/gpfs/dell1/nco/ops/nwprod"];
        last;
    }
}
close VERSION_FILE;

# Special case for the decoders which are currently version-controlled via symlinks
# instead of version files
#readlink 

# Special case for the NCEP Production Libraries which are managed via modulefiles
# and are typically only used for static building and not loaded at runtime


foreach my $key (keys %package_info) {
    print $package_info{$key}[1] . "/" . $key . "." . $package_info{$key}[0] . "\n";
}

# Walk through the transfer template file.  If the $RUN environment variable is
# "nwprod_cleanup":
#  Create a transfer list file with the delete rsync option enabled.  Exclude the
#  latest version (in %package_info) of all packages in the transfer template.
# If $RUN is NOT "nwprod_cleanup":
#  Create a transfer list file that includes all of the latest versions of the
#  packages in the transer template file that are located in the current $NWROOT.
open(TRANSFER_TEMPLATE, $transfer_template);
open(TRANSFER_LIST, ">$transfer_list");
my $printing = 1;
my $missing_packages = 0;
open(MISSING_PACKAGES, ">missing_packages.txt");
print MISSING_PACKAGES "The following packages were not found in any version files.  Please address a.s.a.p.:\n";
close MISSING_PACKAGES;
my $NWROOT = Cwd::abs_path $ENV{'NWROOT'};
while (my $line = <TRANSFER_TEMPLATE>) {
    if ($line =~ /^nwprod\/(\S+)/) {
        my $package = $1;
        if ($ENV{'RUN'} eq "nwprod_cleanup") {
            if ($printing == 1) {
                mkdir "nil";
                print TRANSFER_LIST "nil/ /home/people/nco/www/htdocs/pmb/codes/nw_ENVIR_/\n";
                print TRANSFER_LIST "D\n";
                print TRANSFER_LIST "E\n";
            }
            if (exists $package_info{$package}) {
                my $package_ver = $package;
                $package_ver .= "." . $package_info{$package}[0] if $package_info{$package}[0];
                print TRANSFER_LIST "- /$package_ver\n";
            } else {
                print STDERR "ERROR: $package not found in any version files!\n";
                #system ('err_exit', "$package not found in any version files!");
                $missing_packages++;
                open(MISSING_PACKAGES, ">>missing_packages.txt");
                print MISSING_PACKAGES "- $package\n";
                close MISSING_PACKAGES;
            }
            $printing = 0;
        } else {
            if (exists $package_info{$package} && $package_info{$package}[1] eq $NWROOT) {
                $printing = 1;
                $line =~ s/^nwprod/$ENV{'NWROOT'}/;
                if ($package_info{$package}[0]) {
                    $line =~ s/$package/$package.$package_info{$package}[0]\//g;
                } else {
                    $line =~ s/$package/$package\//g;
                }
                # Only include the active decoders as specified by the symlinks in the decoders directory
                if ($package eq "decoders") {
                    my $decoders_dir = $package_info{$package}[1] . "/$package";
                    opendir(my $dh, $decoders_dir) || print "WARNING: Can't opendir decoders: $!";
                    my @symlinks = grep { -l "$decoders_dir/$_" } readdir($dh);
                    closedir $dh;
                    $line .= "- exec/\n";
                    $line .= "- *polarssl*\n";
                    $line .= "+ /decod_shared/***\n";
                    foreach my $symlink (@symlinks) {
                        $line .= "+ /" . readlink("$decoders_dir/$symlink") . "/***\n";
                    }
                    $line .= "- *\n";
                }
            } else {
                $printing = 0;
            }
        }
    }
    print TRANSFER_LIST $line if ($printing);
}
close TRANSFER_LIST;
close TRANSFER_TEMPLATE;
if ($missing_packages > 0) {
    system ('mail.py -s "RZDM transfer list file requires attention" -v < missing_packages.txt');
}