Clean commit

.gitignore

@@ -0,0 +1,132 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+out_local/
+.ipynb_checkpoints
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

BashScripts/2_clip_to_bbox.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#SBATCH -N 1 #number of nodes
+#SBATCH -n 4 #number of cores per node
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --error=clip.%A.%a.%j.err
+#SBATCH --output=clip.%A.%a.%j.out 
+#SBATCH --array=1-14
+#sleep $((RANDOM%30+1))
+
+#Settables
+OUT_DIR="<path/to/image_dir/planet>/SR_clipped"
+IN_DIR="<path/to/image_dir/planet>/SR_Harmonized"
+#AOI_FILE="/path/to/data_dir/lon_lat_bbox_600m_example.csv"
+AOI_sub="/path/to/data_dir/bbox_AOIs/${SLURM_ARRAY_TASK_ID}.csv"
+
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+
+burntfields clip_images --aoi_file $AOI_sub --in_dir $IN_DIR --out_dir $OUT_DIR
+

BashScripts/2a_make_extent_xml.job

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+#SBATCH -N 1 # number of nodes
+#SBATCH -n 1 # number of cores
+#SBATCH -t 0-08:00 # time (D-HH:MM)
+#SBATCH -o ExtemtFile.%a.%j.out # STDOUT
+#SBATCH -e ExtentFile.%a.%j.err # STDERR
+#SBATCH --job-name="ExtentFile"
+
+################################Settables:
+
+IN_DIR="<path/to/image_dir/planet>/SR_Harmonized"
+OUT_DIR="<path/to/image_dir/planet>/SR_Harmonized"
+TIFLIST="<path/to/data_dir>/missing_xml.csv"
+
+##Activate the virtual environment (which relys on anaconda)
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+
+python MakeExtentXml.py $IN_DIR $OUT_DIR $TIFLIST
+
+deactivate

BashScripts/3_remove_blank_files.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#SBATCH --nodes=1 --ntasks-per-node 12
+#SBATCH --job-name=rmblanksburntfields
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --output=rmBlanks.%j.out # STDOUT
+#SBATCH --error=rmBlanks.%j.err # STDERR 
+
+#Settables:
+OUT_DIR="<path/to/image_dir/planet>/SR_clipped"
+ENDSTRING=".tif"
+
+#Activate the virtual environment (which relys on anaconda)
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+
+burntfields rm_blanks --out_dir $OUT_DIR --endstring $ENDSTRING

BashScripts/4_remove_images.sh

@@ -0,0 +1,19 @@
+#!/bin/bash 
+#SBATCH --nodes=1 --ntasks-per-node 12
+#  this is asking for 1 node, with 12 cores per node
+#SBATCH --job-name=filter_images
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --output=filter_Images.%j.out # STDOUT
+#SBATCH --error=filter_images.%j.err # STDERR 
+
+#Settables:
+IN_DIR="<path/to/image_dir/planet>/SR_clipped"
+OUT_DIR="<path/to/image_dir/planet>/SR_clipped_2"
+REMOVE_FILE="<path/to/data_dir>/cloudy_scenes.txt"
+
+#Activate the virtual environment (which relys on anaconda)
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+
+burntfields rm_images --rm_file $REMOVE_FILE --in_dir $IN_DIR --out_dir $OUT_DIR

BashScripts/5_mask_clouds.sh

@@ -0,0 +1,21 @@
+#!/bin/bash 
+#SBATCH --nodes=1 --ntasks-per-node 12
+#  this is asking for 1 node, with 12 cores per node
+#     the -l is needed on first line if you want to use modules
+#SBATCH --job-name=MaskClouds
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --output=MaskClouds.%j.out # STDOUT
+#SBATCH --error=MaskClouds.%j.err # STDERR 
+
+#Settables:
+OUT_DIR="<path/to/image_dir/planet>/SR_ClipCalMasked"
+IN_DIR="<path/to/image_dir/planet>/SR_clipped"
+TYPE='udm2csorig'
+ENDSTRING='AnalyticMS_SR'
+
+#Activate the virtual environment (which relys on anaconda)
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+
+burntfields mask --in_dir $IN_DIR --out_dir $OUT_DIR --out_type $TYPE --endstring $ENDSTRING

BashScripts/6_clip_to_poly.sh

@@ -0,0 +1,23 @@
+#!/bin/bash 
+#SBATCH -N 1 #number of nodes
+#SBATCH -n 1 #number of cores per node
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --error=clipToPoly.%A.%a.%j.err
+#SBATCH --output=clipToPoly.%A.%a.%j.out 
+#SBATCH --array=1-30
+#sleep $((RANDOM%30+1))
+
+#Settables
+OUT_DIR="<path/to/image_dir/planet>/TS_polys"
+IN_DIR="<path/to/image_dir/planet>/SR_ClipCalMasked"
+SHAPEFILE="<path/to/data_dir>/clean_2Tp_example.shp"
+FILTER="<path/to/data_dir>/bbox_AOIs/${SLURM_ARRAY_TASK_ID}.csv"
+NUM_CORES=1
+STRING='AnalyticMS_SR'
+
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+
+burntfields clip_to_poly --shapefile $SHAPEFILE --shape_filter $FILTER --in_dir $IN_DIR --out_dir $OUT_DIR  --mstring $STRING --num_cores $NUM_CORES
+

BashScripts/7_remove_blank_files2.sh

@@ -0,0 +1,19 @@
+#!/bin/bash  -l
+#SBATCH --nodes=1 --ntasks-per-node 12
+#  this is asking for 1 node, with 12 cores per node
+#     the -l is needed on first line if you want to use modules
+#SBATCH --job-name=rmblanksburntfields
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --output=rmBlanks.%j.out # STDOUT
+#SBATCH --error=rmBlanks.%j.err # STDERR 
+
+#Settables:
+OUT_DIR="<path/to/image_dir/planet>/SR_poly_2"
+ENDSTRING=".tif"
+
+#Activate the virtual environment (which relys on anaconda)
+conda activate conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+
+burntfields rm_blanks --out_dir $OUT_DIR --endstring $ENDSTRING

BashScripts/LOOCV.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# SBATCH -N 1 #number of nodes
+# SBATCH -n 2 #number of cores per node
+# SBATCH --mail-type=END,FAIL
+# SBATCH --mail-user=<your_email>
+# SBATCH --error=LOOCV.%A.%a.%j.err
+# SBATCH --output=LOOCV.%A.%a.%j.out 
+# SBATCH --array=19
+# SBATCH --mem-per-cpu=32G   # memory per cpu-core
+# sleep $((RANDOM%30+1))
+
+#Settables
+OUT_DIR="<path/to/out_dir>/Punjab_LOOCV/Results/${SLURM_ARRAY_TASK_ID}"
+HO_LIST="<path/to/out_dir>/Punjab_LOOCV/HOSets/${SLURM_ARRAY_TASK_ID}.csv"
+LABELS="<path/to/data_dir>/labels_2019_with_bbox_V4.csv"
+ALLFEAT="<path/to/data_dir>/V4_pixelData_COMBO.csv"
+VARIABLES="<path/to/data_dir>/Variables.csv"
+SEED1=8
+SEED2=6888
+DROP_BORDER='True'
+
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+burntfields loocv --out_dir $OUT_DIR --ho_list $HO_LIST --labeled_list $LABELS --variable_path $VARIABLES --all_features $ALLFEAT --seed1 $SEED1 --seed2 $SEED2 --drop_border $DROP_BORDER

BashScripts/LOOCVConsolidate.job

@@ -0,0 +1,20 @@
+#!/bin/bash
+#SBATCH --nodes=1 --ntasks-per-node 1
+#SBATCH --job-name=LOOCVc
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --output=LOOCVc.%j.out # STDOUT
+#SBATCH --error=LOOCVc.%j.err # STDERR 
+
+#Settables:
+
+IN_DIR="<path/to/model_dir>/Rf_model/Results"
+OUT_DIR="<path/to/out_dir>"
+ALLFEAT="<path/to/data_dir>/V4_pixelData_COMBO.csv"
+
+#Activate the virtual environment (which relys on anaconda)
+conda activate venv.burntfields
+
+cd $SLURM_SUBMIT_DIR
+python consolidate_loocv.py $IN_DIR $OUT_DIR $ALLFEAT
+

BashScripts/download_scripts/Planet_dl_porder.job

@@ -0,0 +1,20 @@
+#!/bin/bash  -l
+#SBATCH --nodes=2 --ntasks-per-node 12
+#  this is asking for 2 nodes, with 12 cores per node
+#     the -l is needed on first line if you want to use modules
+#SBATCH --job-name=PlanetDL
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --output=pldown.%j.out # STDOUT
+#SBATCH --error=pldown.%j.err # STDERR 
+sleep $((RANDOM%30+1))
+
+#Settables:
+URL="https://api.planet.com/compute/ops/orders/v2/<order_id>"
+STORAGE="/path/to/image/directory/on/cluster"
+
+#Activate the virtual environment (which relys on anaconda)
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+porder multiproc --url $URL --local $STORAGE
+

BashScripts/download_scripts/Planet_list_porder.job

@@ -0,0 +1,30 @@
+#!/bin/bash  -l
+#SBATCH --nodes=1 --ntasks-per-node 12
+#  this is asking for 1 node, with 12 cores
+#     the -l is needed on first line if you want to use modules
+#SBATCH --job-name=PlanetList
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=<your_email>
+#SBATCH --output=pllist.%j.log # STDOUT
+#SBATCH --error=pllist.%j.err # STDERR 
+#Copy log file into permanent location:
+
+#Settables:
+AOI="<path/todata_dir>/AOI_single.geojson"
+START="2019-10-01"
+END="2019-12-10"
+ITEM="PSScene4Band"
+ASSET="analytic"
+OUTFILE="/path/to/location/for/order_list/PlanetList.csv"
+NUMMAX=2000
+CLOUDMAX=10
+OVERLAP=50
+
+#Activate the virtual environment (which relys on anaconda)
+source ~/.bashrc
+conda activate venv.burntfields
+cd $SLURM_SUBMIT_DIR
+porder idlist --input $AOI --start $START --end $END --item $ITEM --asset $ASSET --number $NUMMAX --outfile $OUTFILE --cmax $CLOUDMAX --overlap $OVERLAP
+
+cp -f ~/code/bash/pllist.${SLURM_JOB_ID}.log /path/to/location/for/order_list/ListSummary.log
+