example-config.yaml

# Configuration file for the NEMO Fetch tool - an automatic downloader of files.
# (In YAML format https://en.wikipedia.org/wiki/YAML )

# An explanation of most of these options is available in the file README.md
#  -> Viewable here: http://rhe-neo-dev-repo.dev.lan/git/blob/pma!fetch.git/master/README.md

# The work directory (for log and lock files):
directory: /data/fetch

# Notification settings (for errors):
notify:
  email: ['jeremy.hooke@ga.gov.au']

# Message bus config, for announcing arrivals. Optional.
messaging:
  host: rhe-neo-dev01.dev.lan
  # virtual_host:
  username: fetch
  password: fetch

# Logging level for modules (any python module can be added here)
log:
  fetch: DEBUG
  neocommon: DEBUG
  neocommon.files: INFO

# Download rules:
rules:
  BRDF from NCI:
    schedule: '0 9 * * 6'
    source: !rsync
      source_host: lpgs@r-dm.nci.org.au
      source_path: /g/data/u39/public/data/modis/lpdaac-mosaics-cmar/v1-hdf4/aust/MCD43A1.005/*
      target_path: /eoancillarydata/BRDF/CSIRO_mosaic

  LS5 CPF:
    schedule: '0 * * * *'
    source: !rss
      url: https://landsat.usgs.gov/L5CPFRSS.rss
      target_dir: /eoancillarydata/sensor-specific/LANDSAT5/CalibrationParameterFile

  LS7 CPF:
    schedule: '10 * * * *'
    source: !rss
      url: http://landsat.usgs.gov/L7CPFRSS.rss
      target_dir: /eoancillarydata/sensor-specific/LANDSAT7/CalibrationParameterFile

  # Available 2-4 hours after acquisition
  LS8 BPF:
    schedule: '*/15 * * * *'
    source: !rss
      url: http://landsat.usgs.gov/bpf.rss
      target_dir: /eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/{year}/{month}
      # Extract the year and month from each filename to use in target_dir
      #    Example filenames: 'LT8BPF20141028232827_20141029015842.01'
      #                       'LO8BPF20141028174324_20141028183542.02'
      filename_transform: !regexp-extract 'L[TO]8BPF(?P<year>[0-9]{4})(?P<month>[0-9]{2})(?P<day>[0-9]{2}).*'

  LS8 CPF:
    schedule: '*/30 * 1 1,4,7,10 *'
    source: !rss
      url: http://landsat.usgs.gov/cpf.rss
      target_dir: /eoancillarydata/sensor-specific/LANDSAT8/CalibrationParameterFile

  LS8 TLE:
    schedule: '20 * * * *'
    source: !rss
      url: http://landsat.usgs.gov/exchange_cache/outgoing/TLE/TLE.rss
      target_dir: /eoancillarydata/sensor-specific/LANDSAT8/TLE/LS8_YEAR/{year}
      # Extract the year from each filename to use in target_dir.
      # Example filename: '506_MOE_ACQ_2014288120000_2014288120000_2014288123117_OPS_TLE.txt'
      filename_transform: !regexp-extract '([A-Z0-9]+_){3}(?P<year>[0-9]{4})(?P<jul>[0-9]{3})[0-9]{6}.*_OPS_TLE.txt'
      beforehand: !http-post
        url: https://landsat.usgs.gov/up_login.php
        params:
            username: australia
            password: fake-password


  Modis Att-Ephem:
    schedule: '20 */2 * * *'
    source: !date-range
      # Download the last three days.
      start_day: -3
      end_day: 0
      overridden_properties:
        url: http://oceandata.sci.gsfc.nasa.gov/Ancillary/Attitude-Ephemeris/{year}/{julday}
        target_dir: /eoancillarydata/sensor-specific/MODIS/ancillary/{year}/{julday}
      using: !http-directory
        name_pattern: '[AP]M1(ATT|EPH).*'
        target_dir: ''
        url: ''

  Modis GDAS:
    schedule: '3 0-23/2 * * *'
    source: !date-range
      # Download from one day ago (-1) to tomorrow (1):
      start_day: -1
      end_day: 1
      overridden_properties:
        source_dir: /pub/eosdb/ancillary/{year}_{month}_{day}_{julday}
        target_dir: /eoancillarydata/sensor-specific/MODIS/ancillary/{year}/{julday}
      using: !ftp-directory
        hostname: ftp.ssec.wisc.edu
        # Get any file starting with 'gdas'
        name_pattern: gdas.*
        # Overridden via the date range patterns
        source_dir: ''
        target_dir: ''

  Modis GFS:
    schedule: '53 0-23/2 * * *'
    source: !date-range
      # Download from one day ago (-1) to tomorrow (1):
      start_day: -1
      end_day: 1
      overridden_properties:
        source_dir: /pub/eosdb/ancillary/{year}_{month}_{day}_{julday}/forecast
        target_dir: /eoancillarydata/sensor-specific/MODIS/ancillary/{year}/{julday}/forecast
      using: !ftp-directory
        hostname: ftp.ssec.wisc.edu
        # Get any file starting with 'gfs'
        name_pattern: gfs.*
        # Overridden via the date range patterns
        source_dir: ''
        target_dir: ''

  Modis TLE:
    schedule: '30 0-23/2 * * *'
    source: !ftp-files
      hostname: is.sci.gsfc.nasa.gov
      paths:
      - /ancillary/ephemeris/tle/drl.tle
      - /ancillary/ephemeris/tle/norad.tle
      target_dir: /eoancillarydata/sensor-specific/MODIS/tle
      # Prepend the current date to the output filename (eg. '20141024.norad.tle')
      filename_transform: !date-pattern '{year}{month}{day}.{filename}'

  Modis utcpole-leapsec:
    schedule: '0 7 * * mon'
    source: !http-files
      urls:
      - http://oceandata.sci.gsfc.nasa.gov/Ancillary/LUTs/modis/utcpole.dat
      - http://oceandata.sci.gsfc.nasa.gov/Ancillary/LUTs/modis/leapsec.dat
      target_dir: /eoancillarydata/sensor-specific/MODIS/

  NOAA TLE:
    schedule: '40 0-23/2 * * *'
    source: !ftp-files
      hostname: is.sci.gsfc.nasa.gov
      paths:
      - /ancillary/ephemeris/tle/noaa/noaa.tle
      target_dir: /eoancillarydata/sensor-specific/NOAA/tle
      # Prepend the current date to the output filename (eg. '20141024.noaa.tle')
      filename_transform: !date-pattern '{year}{month}{day}.{filename}'

  NPP GDAS-forecast:
    schedule: '58 */2 * * *'
    source: !date-range
      # Repeat between 1 day ago to 1 day in the future:
      start_day: -1
      end_day: 1
      overridden_properties:
        target_dir: /eoancillarydata/sensor-specific/NPP/VIIRS/CSPP/anc/cache/{year}_{month}_{day}_{julday}
        url: http://jpssdb.ssec.wisc.edu/ancillary/{year}_{month}_{day}_{julday}
      using: !http-directory
        # Match three file types:
        #   gdas1.pgrb00.1p0deg.20110617_12_000.grib2
        #   NISE_SSMISF17_20110617.HDFEOS
        #   gfs.press_gr.0p5deg_pt.20110617_00_003.npoess.grib2
        name_pattern: (gdas.*\.npoess\.grib2|NISE.*HDFEOS|gfs\.press_gr.*grib2)
        # Overridden via the date range patterns
        target_dir: ''
        url: ''

  NPP LUTS:
    schedule: '0 16 25 * *'
    source: !http-directory
      url: http://jpssdb.ssec.wisc.edu/ancillary/LUTS_V_1_3
      target_dir: /eoancillarydata/sensor-specific/NPP/VIIRS/CSPP/anc/cache/luts

  Water vapour:
    schedule: '0 1 * * *'
    source: !ftp-directory
      hostname: ftp.cdc.noaa.gov
      source_dir: /Datasets/ncep.reanalysis/surface
      # Match filenames such as "pr_wtr.eatm.2014.nc"
      name_pattern: pr_wtr.eatm.20[0-9]{2}.nc
      target_dir: /eoancillarydata/water_vapour/source
    # Convert files to tiff (from netCDF)
    process: !shell
      command: '/usr/local/bin/gdal_translate -a_srs "+proj=latlong +datum=WGS84" {parent_dir}/{filename} {parent_dir}/{file_stem}.tif'
      expect_file: '{parent_dir}/{file_stem}.tif'
#
#-------------------------------
# Examples using the ECMWF API |
#-------------------------------
#
  Invariant GEO Potential:
    schedule: '0 16 25 * *'
    source: !ecmwf-api
        cls: "ei"
        dataset: "interim"
        date: "1989-01-01"
        expver: "1"
        grid: "0.125/0.125"
        levtype: "sfc"
        param: "129.128"
        step: "0"
        stream: "oper"
        time: "12:00:00"
        typ: "an"
        target: /eoancillarydata/ecmwf_data/invar_geo_potential.grib

  Surface Pressure:
    schedule: '0 16 26 * *'
    source: !ecmwf-api
        cls: "ei"
        dataset: "interim"
        date: "2005-01-03/to/2005-01-05"
        area: "0/100/-50/160"
        expver: "1"
        grid: "0.125/0.125"
        levtype: "sfc"
        param: "134.128"
        stream: "oper"
        time: "00:00:00"
        step: "0"
        typ: "an"
        target: /eoancillarydata/ecmwf_data/sp_20050103_to_20050105.grib
    process: !shell
       command: 'python to_geotiff.py -d {parent_dir} {parent_dir}/{file_stem}.grib'
       expect_file: '{parent_dir}/{file_stem}.tif'

  Temperature:
    schedule: '0 17 26 * *'
    source: !ecmwf-api
        cls: "ei"
        dataset: "interim"
        date: "2002-05-05/to/2002-05-05"
        area: "0/100/-50/160"
        expver: "1"
        grid: "0.125/0.125"
        levtype: "sfc"
        param: "130.128"
        stream: "oper"
        time: "10:00:00"
        step: "0"
        typ: "an"
        target: /eoancillarydata/ecmwf_data/temperature_20020505.grib
#
#-------------------------------------------------
# Examples Modis BRDF fetch using authentication |
#-------------------------------------------------
#
  MODIS BRDF:
    schedule: '10 10 * * *'
    source: !date-range
      # Download from one day ago (-1) to tomorrow (1):
      start_day: -14
      end_day: -7
      overridden_properties:
        url: https://e4ftl01.cr.usgs.gov/MOTA/MCD43A1.006/{year}.{month}.{day}
        target_dir: /eoancillarydata/BRDF/MCD43A1.006/{year}.{month}.{day}
      using: !http-directory
        url: ''
        target_dir: ''
        name_pattern: 'MCD43A1\.A[0-9]{7}\.h(2[7-9]|3[0-2])v(09|1[0-3])\.006\.[0-9]{13}\.hdf'
        beforehand: !http-auth
          url: https://urs.earthdata.nasa.gov
          username: australia
          password: fake-password
    process: !shell
      command: 'swfo-convert mcd43a1 h5-md --fname {brdf_base}/{collection}/{ymd}/{basename}{hdf_ext} --outdir /tmp/data/conversion/BRDF/{collection}/{ymd}/ --filter-opts ''{{"aggression": 6}}'' --compression BLOSC_ZSTANDARD'
      input_files: ['^(?P<brdf_base>.*BRDF)/(?P<collection>.*)/(?P<ymd>[0-9]{4}\.[0-9]{2}\.[0-9]{2})/(?P<basename>.*)(?P<hdf_ext>.hdf)(?P<xml_ext>.xml)?', ['{brdf_base}/{collection}/{ymd}/{basename}{hdf_ext}', '{brdf_base}/{collection}/{ymd}/{basename}{hdf_ext}.xml']]
      expect_file: '/tmp/data/conversion/BRDF/{collection}/{ymd}/{basename}.h5'