Adaptation for vm

.travis.yml

@@ -2,11 +2,6 @@ language: python
 python:
   - "2.7"
   - "3.6"
-matrix:
-  include:
-    - python: 3.7
-      dist: xenial
-      sudo: true
 install:
   - sudo apt-get update
   - echo $TRAVIS_PYTHON_VERSION
@@ -17,35 +12,33 @@ install:
     fi
   - bash miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
+  - hash -r
   - conda config --set always_yes yes --set changeps1 no
   - conda config --add channels conda-forge
   - conda update -q conda
-  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six requests scikit-learn scikit-image 'gdal>=2.1,<2.4' pytest-cov coverage pip
+  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal=2.2.4 coveralls pytest-cov coverage=4.5.1 h5py
   - source activate test-environment
   - conda info -a
+  - conda list
+  - conda install pip
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      pip install https://github.com/MarcYin/gp_emulator/archive/master.zip;
+      pip install https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip;
     else
       pip install gp_emulator==1.6.1;
     fi
-  - pip install lightgbm >=2.1.0 coveralls codecov
-  - conda list
+  - pip install coveralls
+  - pip install codecov
 script:
   # Output something every 10 minutes or Travis kills the job
   - while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done &
   - py.test -s --cov
   #Killing background sleep loop
   - kill %1
-  - cp /dev/null SIAC/data/.earthdata_auth
+  #- travis_wait 60 py.test -s --cov
   - coverage xml
-  #- export SIAC_VERSION=$TRAVIS_TAG
-deploy:
-  provider: pypi
-  user: $PYPI_USER
-  password: $PYPI_PASS
-  skip_existing: true
-  on:
-    branch: master
 after_success:
   - codecov
   - coveralls
+  - rm -rf SIAC/emus/*.pkl
+  - rm -rf S2?_MSIL1C*.SAFE
+  - test $TRAVIS_BRANCH = "master" && conda install conda-build && conda install anaconda-client && PKG_NAME=siac && USER=f0xy && OS=$TRAVIS_OS_NAME-64 && mkdir ~/conda-bld && conda config --set anaconda_upload no && export CONDA_BLD_PATH=~/conda-bld && export VERSION=`date +%Y.%m.%d` && conda build . -c conda-forge && ls -lah $CONDA_BLD_PATH/$OS && anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $(ls $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION*.tar.bz2) --force

MANIFEST.in

@@ -1,9 +1,6 @@
 include setup.py
-include SIAC/VERSION
 include SIAC/util/*
 include SIAC/data/sen2cloud_detector.pkl
 include SIAC/data/.earthdata_auth
 include SIAC/spectral_mapping/*
 include SIAC/tests/*.py
-include SIAC/tests/aoi.geojson
-include SIAC/tests/s2_flists.txt

README.md

@@ -8,11 +8,9 @@
 [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac)
 [![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)
 [![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/)
-[![Documentation Status](https://readthedocs.org/projects/siac/badge/?version=latest)](https://siac.readthedocs.io/en/latest/?badge=latest)
 [![codecov](https://codecov.io/gh/MarcYin/SIAC/branch/master/graph/badge.svg?longCache=true&style=flat)](https://codecov.io/gh/MarcYin/SIAC)
 [![Coverage Status](https://coveralls.io/repos/github/MarcYin/SIAC/badge.svg?branch=master)](https://coveralls.io/github/MarcYin/SIAC?branch=master)
 [![Lisence](https://img.shields.io/pypi/l/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)
-[![DOI](https://zenodo.org/badge/117815245.svg)](https://zenodo.org/badge/latestdoi/117815245)
 
 This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects.
 
@@ -25,31 +23,13 @@ This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse
 
 ## Installation:
 
-1. Directly from github 
+Download this repositories either by `git clone git@ github.com:MarcYin/SIAC.git` or download the zip file and unzip it. In the main directory of it excute `pip install .` , or `pip install SIAC` or `conda install -c f0xy siac`
 
-```bash
-pip install https://github.com/multiply-org/atmospheric_correction/archive/master.zip
-```
-
-
-2. Using PyPI
-
-```bash
-pip install SIAC
-```
-
-
-3. Using anaconda
-
-```bash
-conda install -c f0xy -c conda-forge siac
-```
-
-
-To save your time for installing GDAL:
+To save your time for installing GDAL and lightgbm, please use:
 
 ```bash
-conda install -c conda-forge gdal>2.1
+conda install -c conda-forge gdal=2.2.4
+conda install -c conda-forge lightgbm
 ```
 
 

SIAC/SIAC_S2.py

@@ -1,27 +1,49 @@
-import os
-import sys
 import argparse
+import os
 import requests
-import warnings
-warnings.filterwarnings("ignore") 
 import numpy as np
 from glob import glob
-from SIAC.get_MCD43 import get_mcd43
+from SIAC.get_MCD43 import get_mcd43, get_local_MCD43
 from datetime import datetime
 from SIAC.the_aerosol import solve_aerosol
-from SIAC.create_logger import create_logger
 from SIAC.the_correction import atmospheric_correction
 from SIAC.s2_preprocessing import s2_pre_processing
 from SIAC.downloaders import downloader
 from SIAC.multi_process import parmap
+from SIAC.create_logger import create_logger
 from os.path import expanduser
 home = expanduser("~")
 file_path = os.path.dirname(os.path.realpath(__file__))
 
-def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None):
-    if not os.path.exists(file_path + '/emus/'):
-        os.mkdir(file_path + '/emus/')
-    if len(glob(file_path + '/emus/' + 'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12:
+logger = create_logger()
+
+command_2 = ['python', '/home/tonio-bc/projects/multiply/atmospheric_correction/SIAC/SIAC_S2.py',
+           '-f', '/home/tonio-bc/EOData/m1/s2/2017-06-01/30/S/WJ/2017/6/5/0/', '-m', '/home/tonio-bc/EOData/m1/modis/2017-06-01'
+           '-e', '/home/tonio-bc/EOData/m1/emulators', '-c', '/home/tonio-bc/EOData/m1/cams/2017-06-01',
+           '-d', '/home/tonio-bc/EOData/dems/aster_dem.vrt', '-o', 'False',
+           '-a', "'POLYGON((-2.20397502663252 39.09868106889479,-2.1142106223355313 39.09868106889479,"
+                 "-2.1142106223355313 38.94504502508093,-2.20397502663252 38.94504502508093,"
+                 "-2.20397502663252 39.09868106889479))'"]
+
+
+
+def _ensure_dir_format(dir: str):
+    if not dir.endswith('/'):
+        dir += '/'
+    return dir
+
+
+def SIAC_S2(s2_t, dem_vrt, cams_dir, send_back = False, mcd43 = home + '/MCD43/', emu_dir = file_path + '/emus/',
+            vrt_dir = home + '/MCD43_VRT/', download_mcd43 = 'True', aoi = None):
+    logger.info('Starting SIAC')
+    cams_dir = _ensure_dir_format(cams_dir)
+    mcd43 = _ensure_dir_format(mcd43)
+    emu_dir = _ensure_dir_format(emu_dir)
+    vrt_dir = _ensure_dir_format(vrt_dir)
+    if not os.path.exists(emu_dir):
+        emu_dir = file_path + '/emus/'
+        os.mkdir(emu_dir)
+    if len(glob(emu_dir + '/' + 'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12:
         url = 'http://www2.geog.ucl.ac.uk/~ucfafyi/emus/'
         req = requests.get(url)
         to_down = []
@@ -30,20 +52,22 @@ def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home +
                 fname   = line.split('"')[1].split('<')[0]
                 if 'MSI' in fname:
                     to_down.append([fname, url])
-        f = lambda fname_url: downloader(fname_url[0], fname_url[1], file_path + '/emus/')
+        f = lambda fname_url: downloader(fname_url[0], fname_url[1], emu_dir)
         parmap(f, to_down)
+    download_mcd43 = download_mcd43 == 'True'
     rets = s2_pre_processing(s2_t)
     aero_atmos = []
     for ret in rets:
-        ret += (mcd43, vrt_dir, aoi)
+        ret += (dem_vrt, cams_dir, emu_dir, mcd43, vrt_dir, download_mcd43, aoi)
+        #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret
         aero_atmo = do_correction(*ret)
         if send_back:
             aero_atmos.append(aero_atmo)
     if send_back:
         return aero_atmos
 
-def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \
-                  cloud_mask, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None):
+def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile, dem_vrt, cams_dir,
+                  emus_dir, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', download_mcd_43 = True, aoi=None):
 
     if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'):
         if not os.path.exists(home + '/MCD43/'):
@@ -53,39 +77,28 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \
         if not os.path.exists(home + '/MCD43_VRT/'):
             os.mkdir(home + '/MCD43_VRT/')
 
-    base = os.path.dirname(toa_refs[0])
     base = toa_refs[0].replace('B01.jp2', '')
     with open(metafile) as f:
         for i in f.readlines():
             if 'SENSING_TIME' in i:
                 sensing_time = i.split('</')[0].split('>')[-1]
                 obs_time = datetime.strptime(sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ')
             if 'TILE_ID' in i:
-                sat  = i.split('</')[0].split('>')[-1].split('_')[0]
-                tile = i.split('</')[0].split('>')[-1]
-    log_file = os.path.dirname(metafile) + '/SIAC_S2.log'
-    logger = create_logger(log_file)
-    logger.info('Starting atmospheric corretion for %s'%tile)
-    if not np.all(cloud_mask):
-        handlers = logger.handlers[:]
-        for handler in handlers:
-            handler.close()
-            logger.removeHandler(handler)
-        get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir, log_file = log_file)
-        #logger = create_logger(log_file)
+                sat = i.split('</')[0].split('>')[-1].split('_')[0]
+    if download_mcd_43:
+        get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir)
     else:
-        logger.info('No clean pixel in this scene and no MCD43 is downloaded.')
+        get_local_MCD43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir)
     sensor_sat = 'MSI', sat
     band_index  = [1,2,3,7,11,12]
     band_wv    = [469, 555, 645, 859, 1640, 2130]
     toa_bands   = (np.array(toa_refs)[band_index,]).tolist()
     view_angles = (np.array(view_ang_names)[band_index,]).tolist()
     sun_angles  = sun_ang_name
-    #logger.info('Running SIAC for tile: %s on %s'%(tile, obs_time.strftime('%Y-%M-%d')))
     aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\
                          sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \
-                         file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', \
-                         mcd43_dir=vrt_dir, aoi=aoi, log_file = log_file)
+                         file_path+'/spectral_mapping/', emus_dir=emus_dir, mcd43_dir=vrt_dir, aoi=aoi,
+                         global_dem=dem_vrt, cams_dir=cams_dir)
     aero._solving()
     toa_bands  = toa_refs
     view_angles = view_ang_names
@@ -101,19 +114,18 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \
                                   sun_angles, aot = aot, cloud_mask = cloud_mask,\
                                   tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, \
                                   tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = \
-                                  rgb, emus_dir=file_path+'/emus/', log_file = log_file)
+                                  rgb, emus_dir=emus_dir, global_dem=dem_vrt, cams_dir=cams_dir)
     atmo._doing_correction()
     return aero, atmo
-
-def exe():
+if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Sentinel 2 Atmospheric Correction Excutable')
-    parser.add_argument('-f', "--file_path",      help='Sentinel 2 file path', required=True)
-    parser.add_argument("-m", "--MCD43_file_dir", help="Directory where you store MCD43A1.006 data", default = home + '/MCD43/')
-    parser.add_argument("-v", "--vrt_dir",        help="Where MCD43 vrt stored.",                    default = home + '/MCD43_VRT/')
-    parser.add_argument("-a", "--aoi",            help="Area of Interest.",                          default = None)
+    parser.add_argument('-f', "--file_path",      help='Sentinel 2 file path in the form of AWS', required=True)
+    parser.add_argument("-m", "--MCD43_file_dir", help="Directory where you store MCD43A1.006 data")
+    parser.add_argument("-e", "--emulator_dir",   help="Directory where you store emulators.")
+    parser.add_argument("-d", "--dem",            help="A global dem file, and a vrt file is recommended.")
+    parser.add_argument("-o", "--download",       help="Whether to download MCD 43 Data.")
+    parser.add_argument("-c", "--cams",           help="Directory where you store cams data.")
+    parser.add_argument("-a", "--aoi",            help="Area of Interest.")
     args = parser.parse_args()
-    SIAC_S2(s2_t=args.file_path, mcd43=args.MCD43_file_dir, vrt_dir=args.vrt_dir, aoi=args.aoi)
-
-if __name__ == '__main__':
-    exe()
-
+    SIAC_S2(s2_t=args.file_path, dem_vrt=args.dem, cams_dir=args.cams, mcd43=args.MCD43_file_dir,
+        vrt_dir=args.MCD43_file_dir, download_mcd43=args.download, emu_dir=args.emulator_dir, aoi=args.aoi)

SIAC/__init__.py

@@ -1,6 +1,5 @@
 from .the_aerosol import solve_aerosol
 from .the_correction import atmospheric_correction
-from .atmo_solver import solving_atmo_paras
 from .SIAC_S2 import SIAC_S2
 from .SIAC_L8 import SIAC_L8