#!/bin/csh

#123456789-123456789-123456789-123456789-123456789-123456789-123456789-123456789-
# Run by cron hourly-per-NET via 'dmcque' user crontab on OPS1.
# Run WGET to FTP or HTTP metadata files from remote sites and stage those files in the seed STAGE dir.
# Run it without any args to get errors or warnings for WGET SPECS.

# --------------------------------------------
# Historical notes:
# - 4T AK AM AY BC BK CA CC CI CN CO GS GM HV LD MB NC NE NM NN NP NQ NZ O2 OK PO SF SN TX Y7
# - NETQUAKEs MX UW CI LD GS 
# - USGS/ASL  CU GT IC IU IW NE US N4
# - Oct, Dec 2015 - added BC CI CO GS LD from anss-sis.scsn.org
# - 2016-10-12 Mea - added Y7 for TempNet from OK. 2016-2017
# - 2016-11-30 Rick - added TX
# - 2017-04-80 Rick - added CN and PO
# - 2017-06-01 Rick - added CA
# - 2019-08-19 Un - added 4T (same place as TX)
# - 2019-10-18 Forrest - modify to bring in dataless AND .xml metadata, added standard WGET_DEFAULT_OPTIONS, METADATA_PATTERN.
# - 2019-10-24 Forrest - added switch STAGE_XML_FILES to switch staging of XML files ON(1) or OFF(0).
# - 2020-01-07 Un - added PR
#
# --------------------------------------------
# Comments:
# - The script processes one NET at a time .. and one METADATA_PATTERN.
# - We used to just pull METADATA_PATTERN = 'dataless' metadata files, but as of OCT-2019 we also need to pull '.xml' pattern metadata files.
# - Each NET has a WGET NET directory used to store metadata files matching METADATA_PATTERN pulled from the NETs 1-or-more source(s).
#
# Algorithm:
# - The most-recent-metadata-file (matching METADATA_PATTERN) in the WGET NET directory is checked BEFORE we run WGET.
# - WGET is run for the NET and may or may not pull over newer 'accepted' (i.e., WGET -A "source-pattern") metadata files from 1 or more sources.
# - The most-recent-metadata-file (matching METADATA_PATTERN) in the WGET NET directory is checked AFTER we run WGET.
# - If most-recent-metadata-file changed then newer metadata files (matching METADATA_PATTERN) that arrived are moved to STAGE.
#
# --------------------------------------------
# WARNING: this script uses a potentially trecherous algorithm that can pull metadata files to IRIS but skip over moving them to STAGE if not coded properly.
# Understand why:
#
# - The set of 1 or more WGET 'accept' commands for a NET determine what remote metadata we match; they pull all accepted metadata files into the WGET NET directory.
# - If we add a new WGET source for a NET or change the WGET (-A) accept argument for a NET source
# 	 we may suddenly pull a large number of new matching files - not just NEWER files, but ANY new
# 	 accepted files at the new source that were not yet in the WGET NET directory.
#
# - We add WGET sources: 
#   * when we have a new remote source for a single METADATA_PATTERN (i.e., a new 'dataless' source).
#   * or we have a new METADATA_PATTERN (i.e., '*.xml').
#
# - But only a subset of those files we WGET and pull over will be moved to STAGE - only files newer than the most-recent-metadata-file matching the METADATA_PATTERN will be matched and moved to STAGE!
#   * BAD-CASE:
#     We have SRC-1 for a NET accepting 'dataless' files.
#     We add SRC-2 for the NET also accepting 'dataless' files.
#     WGET runs and gets NEWER files from SRC-1 and ALL files from SRC-2 (since none were in WGET NET).
#     None of the files from SRC-2 that are older than the most-recent-metadata-file pulled from SRC-1 will be pushed to STAGE. 
#
# - It would NOT be OK to run the script for METADATA_PATTERN = 'dataless' and METADATA_PATTERN = '*.xml' in separate runs:
#   * Though the metadata files are stored in the same WGET NET directory, and matched separately in each run,
#     if we are stacking 'WGET -A' commands for the two patterns in the same 'switch' block (as we do now)
#     each METADATA_PATTERN run would WGET both pattern's files, however only files newer than the most-recent-metadata-file for METADATA_PATTERN
#     will be moved to STAGE. 
#   * If we wanted to run separate METADATA_PATTERNs in separate runs we must only run the WGET commands associated with that METADATA_PATTERN
#     in that run, i.e., if pattern is 'dataless', only run 'dataless' accepting WGETs, and run separately the '.xml' pattern with the WGETS accepting '*.xml' files.  
#
# - It IS OK to run the script with one METADATA_PATTERN that includes multiple METADATA_PATTERNs but they have to be merged in one 'extended' (egrep) expression:
#   * i.e., use 'egrep "pattern-1|pattern-2"' for example 'egrep "dataless|*.xml"' to find the most-recent-metadata-file.
#   * The affect will be that the most-recent-file of EITHER pattern (or it could match both patterns, i.e., "NET_STA_dataless.xml")
#     will be used as checkpoint for what files to STAGE, and since we pulled all files,
#     both matches will be properly moved to STAGE and not overlooked.

#set echo
if (! ${?BIN}) then
	setenv BIN  /users/dmcque/ops/WSPPAS
	source $BIN/setup
endif

if ( $#argv == 1 ) then
	set NET = $1
else
	echo "Usages:  $BIN/bin/dlftp.csh NETWORK"
	exit
endif

# ----------------------------------------
# Define context.

# Added %M in case we run this script more than once an hour.
set METADATA_FILE_DATE_TAG = `date +'%Y%m%d%H'`

set LOG_DATE_FMT = '%Y-%m-%d %H:%M:%S'

# Use extended grep to allow multiple patterns.
set GREP_CMD = "egrep"

# These dont appear to be used!
set dct = `whoami`
set email = $ARCHIVER,rick@iris.washington.edu,met@iris.washington.edu

# Directories.
set SEED_OLD_HOME_DIR = /seed/SEEDS_FOR_ORACLE
set SEED_WGET_DIR = $SEED_OLD_HOME_DIR/WGET
set WGET_NET_DIR = $SEED_WGET_DIR/$NET
set SEED_NEW_HOME_DIR = /seed/metadata_utilities

# We have the old and new STAGE dirs to choose from.
set SEED_STAGE_DIR_DATALESS = $SEED_OLD_HOME_DIR/TODO
set SEED_STAGE_DIR_SXML = $SEED_NEW_HOME_DIR/productiondb/station-data/stage

# We have the old dataless and new dataless and xml METADATA_PATTERN to choose from.
set METADATA_PATTERN_DATALESS = "dataless"
set METADATA_PATTERN_SXML = "dataless|xml"

# ================================================
# Choose SEED_STAGEDIR and METADATA_PATTERN to use and whether to STAGE XML files:
# ================================================

set SEED_STAGE_DIR = $SEED_STAGE_DIR_SXML
set METADATA_PATTERN = $METADATA_PATTERN_SXML

# Set OFF(0) or ON(1) - the files are still pulled to IRIS, they are just not staged to the loader.
# Note - when toggling this you need to toggle the ANSS dataless and XML WGETS enable or disabled.
set STAGE_XML_FILES = 1

# Add spacer between runs.
echo ""
echo "`date +'$LOG_DATE_FMT'` - BEGIN WGET for $NET - matching metadata pattern [$METADATA_PATTERN] and pushing new files to [$SEED_STAGE_DIR]"

# ----------------------------------------
# Check locks.

if ( -e $BIN/stop/stop.$HOST.lck || -e $BIN/stop/stop.moveDataless ) then
	echo "`date +'$LOG_DATE_FMT'` - $BIN/stop/stop.$HOST.lck OR $BIN/stop/stop.moveDataless found -- EXIT"	
	exit
endif

# ----------------------------------------
# Ensure directories.

test -d $SEED_STAGE_DIR
if ( $status != 0 ) mkdir -p $SEED_STAGE_DIR

test -d $WGET_NET_DIR
if ( $status != 0 ) mkdir -p $WGET_NET_DIR

# Work in the WGET NET directory - WGET pulls files here.
cd $WGET_NET_DIR

# ----------------------------------------
# Check for previous DL files.

# Grep '-i' ignores case.
# Grep '-c' returns count.
set num_prev_metadata_files = ` ls $WGET_NET_DIR | $GREP_CMD -c -i "$METADATA_PATTERN" `

set prev_metadata_fileage_minutes = 0
set prev_metadata_filename = ""

if ( $num_prev_metadata_files > 0 ) then

	# Get the most recent file pulled over.
	# Grep '-i' ignores case.
	set prev_metadata_filename = ` ls -ltr | $GREP_CMD -i "$METADATA_PATTERN" | tail -1 | awk '{print $NF}' `

	# Get its age in minutes.
    set prev_metadata_fileage_minutes = `$BIN/bin/minutes_since $prev_metadata_filename`

	echo "`date +'$LOG_DATE_FMT'` - Newest $METADATA_PATTERN in $WGET_NET_DIR before WGET $prev_metadata_filename age $prev_metadata_fileage_minutes"

else

	echo "`date +'$LOG_DATE_FMT'` - No previous $METADATA_PATTERN exists in $WGET_NET_DIR"

endif

###################################################################
# Start metadata file download.
# Filename formats for all NetQuake networks changed on 01/23/2015 from *_$NET_.dataless  to *_$NET.dataless
# NetQuake Networks: AK AY CI GS HV LD MB MX NC NE NM NN NP NQ UW
# FRENCH Networks from IPGP: G GL MQ PF WI XA
# These French lines were added by Rick on May 10, 2016
###################################################################

# wget arguments:
# -P  : directory Prefix
# -A  : Accept pattern
# -m  : Turn on options suitable for mirroring: recursion, time-stamping, infinite recursion depth, keep FTP dir listings.
# -r  : Turn on recursive retrieving. The default maximum depth is 5. 
# -N  : Turn on time-stamping.
#       With this option, for each file it intends to download, Wget will check whether a local file of the same name exists.
#       If it does, and the remote file is not newer, Wget will not download it. 
# -nd : Do not create a hierarchy of directories when retrieving recursively.
# -np : Do not ever ascend to the parent directory when retrieving recursively.
# -nv : Turn off verbose without being completely quiet.
# -e  : Execute a wget command as if set in .wgetrc.

set WGET_DEFAULT_OPTIONS = "-N -m -nd -np -nv -r"
set WGET_ROBOTS_OFF = "-e robots=off"

set SOURCE_SXML_ANSS      = "https://files.anss-sis.scsn.org/production/FDSNstationXML"

set SOURCE_DATALESS_ANSS  = "http://files.anss-sis.scsn.org/production/dataless"
set SOURCE_DATALESS_IPGP  = "http://datacenter.ipgp.fr/metadata"
set SOURCE_DATALESS_OGSOU = "http://wichita.ogs.ou.edu/seed"
set SOURCE_DATALESS_NRCAN = "ftp://ftp.seismo.nrcan.gc.ca/dataless"
set SOURCE_DATALESS_NETQUAKES = "ftp://ehzftp.wr.usgs.gov/luetgert/netquakes/IRIS/"
set SOURCE_DATALESS_ASL = "ftp://ftpext.cr.usgs.gov/pub/cr/nm/albuquerque/ASL/pub/metadata/FDSNstationXML/"

switch ( $NET )

    case 4T:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"4T.*.dataless" ftp://rtserve.beg.utexas.edu/pub/
    breaksw

    case CA:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"CA*.dataless" http://sismocat.icgc.cat/dataless/dataless.php
    breaksw

    case AG:
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"AG_*.dataless" $SOURCE_DATALESS_ANSS/AG/
    ## 2019-10-29 Tim added anss AG.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"AG_*.xml" $SOURCE_SXML_ANSS/AG/ 
 
    breaksw

    case AK:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_AK.dataless" $SOURCE_DATALESS_NETQUAKES
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"AK_*.dataless" ftp://giseis.alaska.edu/pub/USERS/natasha/AK_dataless
    breaksw

    case AM:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"Dataless.AM.*.seed" $SOURCE_DATALESS_OGSOU 
    breaksw

	case AY:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_AY.dataless" $SOURCE_DATALESS_NETQUAKES
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_AY.dataless" $SOURCE_DATALESS_NRCAN/AY/
	breaksw

	case BC:
	## 2015-12-2 Anh requested by Ellen and ok by Rick
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"BC_*.dataless" $SOURCE_DATALESS_ANSS/BC/
    ## 2019-10-29 Tim added anss BC.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"BC_*.xml" $SOURCE_SXML_ANSS/BC/ 
	breaksw

	case BK:
	### 2019-10-29 Tim noticed that we are not pulling data BK data from ANSS. BK is posted to ANSS. Is NCEDC authoritative?
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"BK.dataless.seed" $WGET_ROBOTS_OFF http://www.ncedc.org/ftp/pub/doc/BK.info/
	breaksw

	case C0:
    ## 2019-04/04 Rick adds after getting note from Kyren Bogolub in Colorado
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"C0_*.dataless" $SOURCE_DATALESS_ANSS/C0/
    ## 2019-10-29 Tim added anss C0.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"C0_*.xml" $SOURCE_SXML_ANSS/C0/ 
	breaksw

	case CC:
    ## 2019-05-28 Rick adds CC after Wes Thelon says he'll use this anss builder
    ## Rick  takes out the -r recursive option, timestamping related
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR -N -nd -np -nv -r -A"CC_*.dataless" $SOURCE_DATALESS_ANSS/CC/
    ## 2019-10-29 Tim added anss CC.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"CC_*.xml" $SOURCE_SXML_ANSS/CC/  
    breaksw

	case CI:
	## 2015-10-22 Anh requested by Ellen and ok by Rick
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.CI.*" http://www.data.scec.org/ftp/stations/seed/
	## back out due to BLK61 empty as an example of station ABL
    ## Rick  takes out the -m mirroring option, timestamping related
    ## Rick  takes out the -r recursive option, timestamping related
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
	# wget -P $WGET_NET_DIR -N -nd -np -nv -r -A"CI_*.dataless" $SOURCE_DATALESS_ANSS/CI/
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_CI.dataless" $SOURCE_DATALESS_NETQUAKES
    ## 2019-10-29 Tim added anss CI.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"CI_*.xml" $SOURCE_SXML_ANSS/CI/ 
	breaksw

	case CN:
    # Rick added 3/8/2017
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"CN*.dataless" $SOURCE_DATALESS_NRCAN/CN/
	breaksw

	case CO:
    ## 2015-12-2 Anh requested by Ellen and ok by Rick
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"CO_*.dataless" $SOURCE_DATALESS_ANSS/CO/
    ## 2019-10-29 Tim added anss CO.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"CO_*.xml" $SOURCE_SXML_ANSS/CO/ 
	breaksw

    case CU:
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.CU_*.seed" ftp://aslftp.cr.usgs.gov/pub/dataless
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"CU_*.xml" $SOURCE_DATALESS_ASL/CU/
    breaksw

	case ET:
	# Rick adds 4-26-2017
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"ET_*.dataless" $SOURCE_DATALESS_ANSS/ET/
    ## 2019-10-29 Tim added anss ET.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"ET_*.xml" $SOURCE_SXML_ANSS/ET/ 
    breaksw

	case G:
    ## 2016-05-10 Rick: Worked with Constanza @ GEOSCOPE to get these for volcano networks in Caribbean 
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.G.*.seed" $SOURCE_DATALESS_IPGP/G/
	breaksw

	case GL:
    ## 2016-05-10 Rick: Worked with Constanza @ GEOSCOPE to get these for volcano networks in Caribbean 
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.GL.*.seed" $SOURCE_DATALESS_IPGP/GL/
	breaksw

	case GM:
	# Rick adds on 4-26-2017
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"GM_*.dataless" $SOURCE_DATALESS_ANSS/GM
    ## 2019-10-29 Tim added anss GM.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"GM_*.xml" $SOURCE_SXML_ANSS/GM/ 
    breaksw

	case GS:
	# Rick adds anss repo on July 15, 2019
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_GS.dataless" $SOURCE_DATALESS_NETQUAKES
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"GS_*.dataless" $SOURCE_DATALESS_ANSS/GS
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.GS*.seed" ftp://aslftp.cr.usgs.gov/pub/dataless
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.GS.*" http://www.data.scec.org/ftp/stations/seed/
    ## 2019-10-29 Tim added anss GS.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"GS_*.xml" $SOURCE_SXML_ANSS/GS/ 
	breaksw

    case GT:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.GT_*.seed" ftp://aslftp.cr.usgs.gov/pub/dataless
    breaksw

	case HV:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_HV.dataless" $SOURCE_DATALESS_NETQUAKES
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*.dataless.seed"  ftp://ftpext.usgs.gov/pub/wr/hi/kilauea/SEED/
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"HV*.dataless" $SOURCE_DATALESS_ANSS/HV/
    ## 2019-10-29 Tim added anss HV.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"HV_*.xml" $SOURCE_SXML_ANSS/HV/ 
	breaksw

    case IC:
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.IC_*.seed" ftp://aslftp.cr.usgs.gov/pub/dataless
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"IC_*.xml" $SOURCE_DATALESS_ASL/IC/
    breaksw

    case II:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"II_*.*.dataless" $WGET_ROBOTS_OFF  http://ida.ucsd.edu/pickup/dataless
    breaksw

    case IU:
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.IU_*.seed" ftp://aslftp.cr.usgs.gov/pub/dataless
	# Alteration by RICK on Sept 9, 2018, to use this ftp address provided by Tyler
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.IU_*.seed" ftp://ftpext.cr.usgs.gov/pub/cr/nm/albuquerque/ASL/pub/dataless/
    breaksw

    case IW:
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.IW_*.seed" ftp://hazards.cr.usgs.gov/ANSS/dataless
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"IW_*.dataless" $SOURCE_DATALESS_ANSS/IW/
    ## 2019-10-29 Tim added anss IW.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"IW_*.xml" $SOURCE_SXML_ANSS/IW/ 
    breaksw

	case LD:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_LD.dataless" $SOURCE_DATALESS_NETQUAKES
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"LD*dataless"   $SOURCE_DATALESS_ANSS/LD/
    ## 2019-10-29 Tim added anss LD.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"LD_*.xml" $SOURCE_SXML_ANSS/LD/ 
	breaksw

    case MB:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_MB.dataless" $SOURCE_DATALESS_NETQUAKES
    breaksw

	case MQ:
    ## 2016-05-10 Rick: Worked with Constanza @ GEOSCOPE to get these for volcano networks in Caribbean 
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.MQ.*.seed" $SOURCE_DATALESS_IPGP/MQ/
	breaksw

    case MX:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_MX.dataless" $SOURCE_DATALESS_NETQUAKES
	## 2015-12-3 Anh requested by Ellen and ok by Rick
    ## 2019-10-28 Tim turned off the wget dataless path for MX and we are recieving only xml from MX. 
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"MX_*.dataless" $SOURCE_DATALESS_ANSS/MX/	 
    ## 2019-10-28 Forrest and Tim use SB as test case for xml loader piping. 
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"MX_*.xml" $SOURCE_SXML_ANSS/MX/ 
    breaksw

    case N4:
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"N4*.dataless" $SOURCE_DATALESS_ANSS/N4/
    ## 2019-10-29 Tim added anss N4.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"N4_*.xml" $SOURCE_SXML_ANSS/N4/ 
    breaksw

	case NC:
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NC*.dataless" $SOURCE_DATALESS_ANSS/NC/
    ## 2019-10-29 Tim added anss NC.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NC_*.xml" $SOURCE_SXML_ANSS/NC/ 
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_NC.dataless" $SOURCE_DATALESS_NETQUAKES
	breaksw

	case NE:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_NE.dataless" $SOURCE_DATALESS_NETQUAKES
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.NE_*.seed" ftp://hazards.cr.usgs.gov/ANSS/dataless
	breaksw

    case NM:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_NM.dataless" $SOURCE_DATALESS_NETQUAKES
	# Rick adds on 4-26-2017
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NM*.dataless" $SOURCE_DATALESS_ANSS/NM/
    ## 2019-10-29 Tim added anss NM.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NM_*.xml" $SOURCE_SXML_ANSS/NM/ 
    breaksw

	case NN:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_dataless" http://crack.seismo.unr.edu/~glenn/ftp/Dataless/NN/
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_NN.dataless" $SOURCE_DATALESS_NETQUAKES
	breaksw

	case NP:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_NP.dataless" $SOURCE_DATALESS_NETQUAKES
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NP_ENB01.dataless" $SOURCE_DATALESS_ANSS/NP/
	# Un adds on 6/13/2019 for Jeff Brody, USGS
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NP.dataless.seed" http://www.ncedc.org/ftp/pub/doc/NP.info/
    ## 2019-10-29 Tim added anss NP.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NP_*.xml" $SOURCE_SXML_ANSS/NP/ 
	breaksw

	case NQ:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_NQ.dataless" $SOURCE_DATALESS_NETQUAKES
	breaksw

	case NW:
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
	# wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NW*.dataless" $SOURCE_DATALESS_ANSS/NW/
    ## 2019-10-29 Tim added anss NW.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"NW_*.xml" $SOURCE_SXML_ANSS/NW/ 
	breaksw

	case NZ:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*nz.dataless.seed.iris"  ftp://ftp.geonet.org.nz/seed/DATALESS/ 
	breaksw

    case O2:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"Dataless.O2.*.seed" $SOURCE_DATALESS_OGSOU
    breaksw

	case OK:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"Dataless.OK.*.seed" $SOURCE_DATALESS_OGSOU
    breaksw

    case PF:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.PF.*.seed" $SOURCE_DATALESS_IPGP/PF/
	breaksw

	case PO:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"PO.dataless" $SOURCE_DATALESS_NRCAN/PO/
	breaksw

	case PR:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"PR_*.xml" $SOURCE_SXML_ANSS/PR/
	breaksw

	case RB:
	## 2015-12-3 Anh requested by Ellen and ok by Rick
	## Rick turns this off on Friday April 19, 2019, after not getting timeseries data, and metadata incomplete and needing removal.
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"RB_*.dataless" $SOURCE_DATALESS_ANSS/RB/
    ## 2019-10-29 Tim added anss RB.xml to metadata loader path.
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"RB_*.xml" $SOURCE_SXML_ANSS/RB/ 
	breaksw
	
	case SB:
	## 2017-07-11 Rick adds this after finding out its on anss
	## 2017-07-12 Rick removes the -m "mirroring" option,trying to fix timestamp messages in log?
    ## 2019-10-28 Forrest and Tim use SB as test case for xml loader piping. 
	# 2019-10-30 Forrest set STAGE_XML_FILES = 0 means dataless pull is off in favor of xml pull
    # wget -P $WGET_NET_DIR -N -nd -np -nv -r -A"SB_*.dataless" $SOURCE_DATALESS_ANSS/SB/
    wget -P $WGET_NET_DIR -N -nd -np -nv -r -A"SB_*.xml"      $SOURCE_SXML_ANSS/SB/
    breaksw

	case SF:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"SF.dataless.seed" $WGET_ROBOTS_OFF  http://www.ncedc.org/ftp/pub/doc/SF.info/
	breaksw

	case SN:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_dataless" http://crack.seismo.unr.edu/~glenn/ftp/Dataless/SN/
	breaksw

	case TX:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"TX.*.dataless" ftp://rtserve.beg.utexas.edu/pub/
	breaksw

    case US:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.US_*.seed" ftp://hazards.cr.usgs.gov/ANSS/dataless
    breaksw

    case UU:
    #wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"DATALESS.US_*.seed" ftp://hazards.cr.usgs.gov/ANSS/dataless
    wget --no-check-certificate -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_*.dataless" https://quake.utah.edu/dataless/
    breaksw

	case UW:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"*_UW.dataless" $SOURCE_DATALESS_NETQUAKES
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"UW_*.xml" $SOURCE_SXML_ANSS/UW
	breaksw

	case WI:
    ## 2016-05-10 Rick: Worked with Constanza @ GEOSCOPE to get these for volcano networks in Caribbean 
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.WI.*.seed" $SOURCE_DATALESS_IPGP/WI/
	breaksw

	case XA:
    ## 2016-05-10 Rick: Worked with Constanza @ GEOSCOPE to get these for Apollo lunar sets
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.XA.*.seed" $SOURCE_DATALESS_IPGP/XA/
	breaksw

	case XB:
    ## 2019-04-24 Rick: Worked with Constanza @ GEOSCOPE to get these for Mars-InSight 
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"dataless.XB.*.seed" $SOURCE_DATALESS_IPGP/XB/
	breaksw

	case Y7:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"Dataless.Y7.*.seed" $SOURCE_DATALESS_OGSOU
    breaksw

	case Y9:
	wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"Dataless.Y9.*.seed" $SOURCE_DATALESS_OGSOU
    breaksw

	case ZP:
    wget -P $WGET_NET_DIR $WGET_DEFAULT_OPTIONS -A"Dataless.ZP.*.seed" $SOURCE_DATALESS_OGSOU
    breaksw

# This endsw comes at the END of new networks list so be careful when adding new networks to the list
endsw

# Dataless download is done.
echo "`date +'$LOG_DATE_FMT'` - END WGET for $NET"

# Anh 08/04/2016 - add this to solve the "Permission Denied"
# Changes permissions on downloaded metadata files.

chmod 755 $WGET_NET_DIR/* 

# ----------------------------------------
# Scan metadata files for NET and determine which are new and can load.

set METADATA_TO_LOAD_FILE = $SEED_WGET_DIR/need2load.$NET.$METADATA_FILE_DATE_TAG

if ( $num_prev_metadata_files > 0 ) then

	# Get the newest metadata file and its age.
	# Grep '-i' ignores case.
	set new_metadata_filename = ` ls -ltr | $GREP_CMD -i "$METADATA_PATTERN" | tail -1 | awk '{print $NF}' `
	set new_metadata_fileage_minutes = `$BIN/bin/minutes_since $new_metadata_filename`

	echo "`date +'$LOG_DATE_FMT'` - Newest $METADATA_PATTERN in $WGET_NET_DIR after WGET is $new_metadata_filename age $new_metadata_fileage_minutes"

	# If newest file after download is the same age as the previous newest file then there are no new files.
	if ( $new_metadata_fileage_minutes >= $prev_metadata_fileage_minutes ) then
		echo "`date +'$LOG_DATE_FMT'` - No new $METADATA_PATTERN for $NET -- exit"
		exit
	endif

	# Otherwise there are newer files.
	# Grep '-i' ignores case.
	foreach file ( ` ls | $GREP_CMD -i "$METADATA_PATTERN" ` )

		# Save files newer in age than the previous newest file (from before WGET).
		set fileage_minutes = `$BIN/bin/minutes_since $file`
		if ( $fileage_minutes < $prev_metadata_fileage_minutes ) then
			echo ./$file >> $METADATA_TO_LOAD_FILE 
		endif
	end

else

	# Otherwise there were no previous metadata files and we can load all metadata files that are in the dir.
	ls > $METADATA_TO_LOAD_FILE

endif
 
# ----------------------------------------
# Copy new metadata files to seed STAGE dir.

# The test utility evaluates the condition and indicates the result of the evaluation by its exit status.
# An exit status of zero indicates that the condition evaluated as true and an exit status of 1 indicates
# that the condition evaluated as false.

test -s $METADATA_TO_LOAD_FILE
if ( $status == 0 ) then

	foreach metadata_filename ( ` cat $METADATA_TO_LOAD_FILE | $GREP_CMD -v CI_online_epochs.dataless ` )

		# 'test -s file' tests file is non-zero size so a value of 0 means true and its not 0-size. 
		test -s $metadata_filename
		if ( $status == 0 ) then

			if ( $metadata_filename !~ *.xml || $STAGE_XML_FILES ) then

				# Copy the non-empty metadata file to the seed STAGE dir with date tag added.
				# The original metadata file remains in the directory to be cleaned up later.
				set dest_filename = $metadata_filename
				set dest_dir = $SEED_STAGE_DIR

				if ( $metadata_filename =~ *.xml ) then

					# Add date tag before ".xml" extension.
    				set dest_filename = ` echo $dest_filename | sed 's/\.xml$//' `
    				set dest_filename = $dest_filename.$METADATA_FILE_DATE_TAG.xml

				else

					# Add date tag at end.
    				set dest_filename = $metadata_filename.$METADATA_FILE_DATE_TAG

				endif

				# SXML loader requires an atomic move - 'cp' may not be fast enough so use 'mv'.
				cp -p $metadata_filename $dest_filename
				mv $dest_filename $dest_dir

				echo "`date +'$LOG_DATE_FMT'` - Staged new $METADATA_PATTERN for $NET file $metadata_filename - copied as $dest_filename to $dest_dir"

				# FST - turned this code off as its just deleted below - not sure what it was for.
				# echo $metadata_filename.$METADATA_FILE_DATE_TAG >> $SEED_OLD_HOME_DIR/sent2load.$NET.$METADATA_FILE_DATE_TAG

			else 

				echo "`date +'$LOG_DATE_FMT'` - Did NOT stage $NET file $metadata_filename - its XML and STAGE_XML_FILES is $STAGE_XML_FILES"

			endif

		else

			# Dataless file was empty.

			# FST - turned this off as its their problem if the dateless file was empty.
			# echo $metadata_filename.$METADATA_FILE_DATE_TAG >> $SEED_OLD_HOME_DIR/empty.$NET.$METADATA_FILE_DATE_TAG

		endif
	end


	# Not clear what this does - list stage dir contents to STDOUT?
	ls -ltr $SEED_STAGE_DIR | $GREP_CMD -i "$METADATA_PATTERN"

endif

# ----------------------------------------
# Clean up.

rm -f $METADATA_TO_LOAD_FILE

# FST - turned this code off as its just deleted and never used - not sure what it was for.
rm -f $SEED_OLD_HOME_DIR/sent2load.$NET.$METADATA_FILE_DATE_TAG 

