#!/usr/bin/perl

use Time::CTime;           # for strftime

#-----------------------------------------------------------
#
# This script downloads water data 20 stations at at time
# from water_data.usgs.gov.  
#
#-----------------------------------------------------------



$dbn_root = $ENV{'DBNROOT'};
if (!defined($dbn_root) || !$dbn_root) {
  print STDERR "\n$0: environment variable DBNROOT not set!\n";
  exit 0;
}
$decd_root = $ENV{'DECDROOT'};
if (!defined($decd_root) || !$decd_root) {
  print STDERR "\n$0: environment variable DECDROOT not set!\n";
  exit 0;
}
$dcom_root = $ENV{'DCOMROOT'};
if (!defined($dcom_root) || !$dcom_root) {
  print STDERR "\n$0: environment variable DCOMROOT not set!\n";
  exit 0;
}
$envir = $ENV{'envir'};
if (!defined($envir) || !$envir) {
  print STDERR "\n$0: environment variable envir not set!\n";
  exit 0;
}

$dbn_config="${dbn_root}/table/dbn_config";
require "$dbn_config" or die "Unable to open $dbn_config";

unshift(@INC, '.');
unshift(@INC, "$include_dir");

require 'dbn_util.pl';
require 'timeout.pl';

sub get_batch;
require 'dbn_util.pl';

$stations_per_query=20;
$station_table="${decd_root}/decod_dcusgs/dictionaries/usgs.tbl";
$yyyymmdd=strftime("%Y%m%d", gmtime(time));
$hhmm=strftime("%H%M", gmtime(time));
$water_data_dir="${dcom_root}/${yyyymmdd}/wtxtbul";
$this_download="${water_data_dir}/usgs_river.${hhmm}";
print "this_download=$this_download\n";

# make the dir

&util'mkdirs($water_data_dir);

if (! open (STATIONS,"<$station_table"))  {
   util'msg($util'ERROR_MSG,"$0: Cannot open station table: $station_table");
   exit 1;
}
$nstations=0;
$nbatch=0;
$batch="";
while ($stationrec=<STATIONS>) {
  next if ( $stationrec =~ /^!/ );  # skip comment lines
  $stationrec =~ /^(\d+)\s.*\s(\w+)\s*$/;
  $station = $1;
  $lastfield = $2;
  # append the last field to the station ID, unless it contains a non-digit 
  # and in which case it isn't a part of the station ID
  $station .= $lastfield unless ( $lastfield =~ /\D/ );
  chomp $station;
  $nstations++;
  if ($nstations gt 1) {
     $batch=$batch . ",";
  }
  $batch=$batch . $station;
  ($nstations <  $stations_per_query) && next;
  $nbatch++;
  get_batch $water_data_dir, $batch, $nbatch, $this_download;
  $batch="";
  $nstations=0
}

# make the query for the leftovers, if any

if ($nstations gt 0) {
  $nbatch++;
  get_batch $water_data_dir, $batch, $nbatch, $this_download;
}

$cmd="${dbn_root}/bin/dbn_alert DATA USGS_RIVER mirror $this_download";
&timeout'run_command_timeout ($cmd, 60);

#
# This subroutine will make the https call to waterdata.usgs.gov for the current batch of stations.
#
sub get_batch {
   $water_data_dir=shift;
   $batch=shift;
   $nbatch=shift;
   $this_download=shift;
   $cmd="wget https://\"waterdata.usgs.gov/nwis/uv?format=rdb&period=1&multiple_site_no=$batch\" -O ${water_data_dir}/stations_$nbatch";
   print "cmd = $cmd\n";
   print "execute and add to $this_download\n";
   &timeout'run_command_timeout ($cmd, 60);
   `cat ${water_data_dir}/stations_$nbatch >> $this_download`;
   unlink "${water_data_dir}/stations_$nbatch";
   print "cmd=$cmd\n";
   print "batch=$batch\n";
   print "this batch goes in  ${water_data_dir}/stations_$nbatch\n";
   print "and was added to $this_download\n";
}

