From 80e6f609d3ad60efc8204932095df78aeef2776c Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 7 Oct 2020 10:22:28 -0400
Subject: [PATCH 01/26] deploy-data: initial version to deploy data from
 multiple git repos

Code is working against included deploy-data.config.sample.yml.
---
 birdhouse/deployment/deploy-data              | 80 +++++++++++++++++++
 .../deployment/deploy-data.config.sample.yml  | 26 ++++++
 2 files changed, 106 insertions(+)
 create mode 100755 birdhouse/deployment/deploy-data
 create mode 100644 birdhouse/deployment/deploy-data.config.sample.yml

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
new file mode 100755
index 000000000..918c3ecc3
--- /dev/null
+++ b/birdhouse/deployment/deploy-data
@@ -0,0 +1,80 @@
+#!/bin/sh
+# Deploy data from git repo(s) to local folder(s).
+#
+# See sample input config in deploy-data.config.sample.yml.
+#
+
+cleanup_on_exit() {
+    set +x
+    echo "
+datadeploy finished START_TIME=$START_TIME
+datadeploy finished   END_TIME=`date -Isecond`"
+}
+
+trap cleanup_on_exit EXIT
+
+
+if [ -z "$DEPLOY_DATA_CHECKOUT_CACHE" ]; then
+    DEPLOY_DATA_CHECKOUT_CACHE="/tmp/deploy-data-clone-cache"
+fi
+
+if [ -z "$DEPLOY_DATA_YQ_IMAGE" ]; then
+    DEPLOY_DATA_YQ_IMAGE="mikefarah/yq:3.3.4"
+fi
+
+CONFIG_YML="$1"; shift
+
+
+yq() {
+    docker run --rm --name yq -v $CONFIG_YML:$CONFIG_YML:ro $DEPLOY_DATA_YQ_IMAGE yq "$@"
+}
+
+
+START_TIME="`date -Isecond`"
+echo "==========
+datadeploy START_TIME=$START_TIME"
+
+set -x
+
+GIT_REPO_URLS="`yq r -p v $CONFIG_YML \[*\].repo_url`"
+REPO_NUM=0
+
+for GIT_REPO_URL in $GIT_REPO_URLS; do
+
+    GIT_BRANCH="`yq r -p v $CONFIG_YML \[$REPO_NUM\].branch`"
+    GIT_CHECKOUT_NAME="`yq r -p v $CONFIG_YML \[$REPO_NUM\].checkout_name`"
+
+    CLONE_DEST="$DEPLOY_DATA_CHECKOUT_CACHE/$GIT_CHECKOUT_NAME"
+    if [ ! -d "$CLONE_DEST" ]; then
+        echo "checkout repo '$GIT_REPO_URL' on branch '$GIT_BRANCH' to '$CLONE_DEST'"
+        git clone $GIT_REPO_URL $CLONE_DEST
+        cd $CLONE_DEST
+        git checkout $GIT_BRANCH
+    else
+        echo "refresh repo '$CLONE_DEST' on branch '$GIT_BRANCH'"
+        cd $CLONE_DEST
+        git clean -fdx  # force, recur dir, also clean .gitignore files and untracked files
+        git fetch --prune --all
+        git checkout --force $GIT_BRANCH  # force checkout to throwaway local changes
+    fi
+
+    SRC_DIRS="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[*\].source_dir`"
+    DIR_NUM=0
+
+    for SRC_DIR in $SRC_DIRS; do
+        DEST_DIR="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[$DIR_NUM\].dest_dir`"
+
+        echo "sync '$SRC_DIR' to '$DEST_DIR'"
+        rsync --recursive --links --checksum --delete \
+            --itemize-changes --human-readable --verbose \
+            $SRC_DIR $DEST_DIR
+
+        DIR_NUM=`expr $DIR_NUM + 1`
+    done
+
+    REPO_NUM=`expr $REPO_NUM + 1`
+
+done
+
+
+# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4
diff --git a/birdhouse/deployment/deploy-data.config.sample.yml b/birdhouse/deployment/deploy-data.config.sample.yml
new file mode 100644
index 000000000..a39d6da04
--- /dev/null
+++ b/birdhouse/deployment/deploy-data.config.sample.yml
@@ -0,0 +1,26 @@
+- repo_url: https://github.com/Ouranosinc/jenkins-master
+  branch: origin/master
+  checkout_name: jenkins-master
+  dir_maps:
+  - source_dir: initial-jenkins-plugins-suggestion
+    dest_dir: /tmp/deploy-data-test-deploy/jenkins-plugins
+
+- repo_url: https://github.com/Ouranosinc/jenkins-config
+  branch: origin/master
+  checkout_name: jenkins-config
+  dir_maps:
+  - source_dir: canarie-presentation
+    dest_dir: /tmp/deploy-data-test-deploy/canarie
+  - source_dir: jcasc
+    dest_dir: /tmp/deploy-data-test-deploy/jcasc
+
+- repo_url: https://github.com/Ouranosinc/pavics-sdi
+  branch: origin/master
+  checkout_name: pavics-sdi
+  dir_maps:
+  - source_dir: docs/source/notebooks
+    dest_dir: /tmp/deploy-data-test-deploy/tutorial-notebooks
+  - source_dir: docs/source/processes
+    dest_dir: /tmp/deploy-data-test-deploy/processes
+  - source_dir: docs/source/tutorials
+    dest_dir: /tmp/deploy-data-test-deploy/tutorials

From b4b8cb5adb49cebcf53a05f7270335464c8de140 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 7 Oct 2020 10:53:00 -0400
Subject: [PATCH 02/26] deploy-data: detect typo in the keys in config file

Prevent silent failure.
---
 birdhouse/deployment/deploy-data | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 918c3ecc3..fdf77551e 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -29,6 +29,14 @@ yq() {
     docker run --rm --name yq -v $CONFIG_YML:$CONFIG_YML:ro $DEPLOY_DATA_YQ_IMAGE yq "$@"
 }
 
+# Empty value could mean typo in the keys in the config file.
+ensure_not_empty() {
+    if [ -z "$*" ]; then
+        echo "ERROR: value empty" 1>&2
+        exit 1
+    fi
+}
+
 
 START_TIME="`date -Isecond`"
 echo "==========
@@ -37,12 +45,15 @@ datadeploy START_TIME=$START_TIME"
 set -x
 
 GIT_REPO_URLS="`yq r -p v $CONFIG_YML \[*\].repo_url`"
+ensure_not_empty "$GIT_REPO_URLS"
 REPO_NUM=0
 
 for GIT_REPO_URL in $GIT_REPO_URLS; do
 
     GIT_BRANCH="`yq r -p v $CONFIG_YML \[$REPO_NUM\].branch`"
+    ensure_not_empty "$GIT_BRANCH"
     GIT_CHECKOUT_NAME="`yq r -p v $CONFIG_YML \[$REPO_NUM\].checkout_name`"
+    ensure_not_empty "$GIT_CHECKOUT_NAME"
 
     CLONE_DEST="$DEPLOY_DATA_CHECKOUT_CACHE/$GIT_CHECKOUT_NAME"
     if [ ! -d "$CLONE_DEST" ]; then
@@ -59,10 +70,12 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
     fi
 
     SRC_DIRS="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[*\].source_dir`"
+    ensure_not_empty "$SRC_DIRS"
     DIR_NUM=0
 
     for SRC_DIR in $SRC_DIRS; do
         DEST_DIR="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[$DIR_NUM\].dest_dir`"
+        ensure_not_empty "$DEST_DIR"
 
         echo "sync '$SRC_DIR' to '$DEST_DIR'"
         rsync --recursive --links --checksum --delete \

From f16b2c9a66d1301ecbcd36b698ecbfdb1c3c1ec4 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 7 Oct 2020 17:18:40 -0400
Subject: [PATCH 03/26] deploy-data: make branch field in config file optional

---
 birdhouse/deployment/deploy-data                   | 2 +-
 birdhouse/deployment/deploy-data.config.sample.yml | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index fdf77551e..ec7a04247 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -50,7 +50,7 @@ REPO_NUM=0
 
 for GIT_REPO_URL in $GIT_REPO_URLS; do
 
-    GIT_BRANCH="`yq r -p v $CONFIG_YML \[$REPO_NUM\].branch`"
+    GIT_BRANCH="`yq r -p v $CONFIG_YML --defaultValue origin/master \[$REPO_NUM\].branch`"
     ensure_not_empty "$GIT_BRANCH"
     GIT_CHECKOUT_NAME="`yq r -p v $CONFIG_YML \[$REPO_NUM\].checkout_name`"
     ensure_not_empty "$GIT_CHECKOUT_NAME"
diff --git a/birdhouse/deployment/deploy-data.config.sample.yml b/birdhouse/deployment/deploy-data.config.sample.yml
index a39d6da04..2782ba183 100644
--- a/birdhouse/deployment/deploy-data.config.sample.yml
+++ b/birdhouse/deployment/deploy-data.config.sample.yml
@@ -1,11 +1,13 @@
 - repo_url: https://github.com/Ouranosinc/jenkins-master
-  branch: origin/master
+  # optional, default "origin/master"
+  # branch:
   checkout_name: jenkins-master
   dir_maps:
   - source_dir: initial-jenkins-plugins-suggestion
     dest_dir: /tmp/deploy-data-test-deploy/jenkins-plugins
 
 - repo_url: https://github.com/Ouranosinc/jenkins-config
+  # optional, default "origin/master"
   branch: origin/master
   checkout_name: jenkins-config
   dir_maps:
@@ -15,7 +17,8 @@
     dest_dir: /tmp/deploy-data-test-deploy/jcasc
 
 - repo_url: https://github.com/Ouranosinc/pavics-sdi
-  branch: origin/master
+  # optional, default "origin/master"
+  # branch:
   checkout_name: pavics-sdi
   dir_maps:
   - source_dir: docs/source/notebooks

From 9f0eed00a30353adf22a455f16f75f70f1181bcf Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 7 Oct 2020 17:26:51 -0400
Subject: [PATCH 04/26] deploy-data: log remote on fetch for future debugging

---
 birdhouse/deployment/deploy-data | 1 +
 1 file changed, 1 insertion(+)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index ec7a04247..266c47f7a 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -64,6 +64,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
     else
         echo "refresh repo '$CLONE_DEST' on branch '$GIT_BRANCH'"
         cd $CLONE_DEST
+        git remote -v  # log remote, should match GIT_REPO_URL
         git clean -fdx  # force, recur dir, also clean .gitignore files and untracked files
         git fetch --prune --all
         git checkout --force $GIT_BRANCH  # force checkout to throwaway local changes

From 162b5f08eabbf0b0ea9387c8054ff23d6f368d47 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 7 Oct 2020 18:33:49 -0400
Subject: [PATCH 05/26] deploy-data: add log output redirection for
 non-interractive use

---
 birdhouse/deployment/deploy-data | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 266c47f7a..527da157d 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -4,6 +4,11 @@
 # See sample input config in deploy-data.config.sample.yml.
 #
 
+if [ ! -z "$DEPLOY_DATA_LOGFILE" ]; then
+    exec >>$DEPLOY_DATA_LOGFILE 2>&1
+fi
+
+
 cleanup_on_exit() {
     set +x
     echo "

From c8a236add74c93a3e12e460009734e38bf728b12 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 7 Oct 2020 18:43:30 -0400
Subject: [PATCH 06/26] deploy-data: ensure config yml file is provided and is
 abs path for docker

---
 birdhouse/deployment/deploy-data | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 527da157d..36cbe6780 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -27,7 +27,15 @@ if [ -z "$DEPLOY_DATA_YQ_IMAGE" ]; then
     DEPLOY_DATA_YQ_IMAGE="mikefarah/yq:3.3.4"
 fi
 
-CONFIG_YML="$1"; shift
+CONFIG_YML="$1"
+if [ -z "$CONFIG_YML" ]; then
+    echo "ERROR: missing config.yml file" 1>&2
+    exit 2
+else
+    shift
+    # Docker volume mount requires absolute path.
+    CONFIG_YML="`realpath "$CONFIG_YML"`"
+fi
 
 
 yq() {

From e6ecc542ad25ca34b1c54c8c95fabcf605b46875 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 7 Oct 2020 19:24:34 -0400
Subject: [PATCH 07/26] deploy-data: use rsync from a docker to be able to run
 inside another docker

---
 birdhouse/deployment/deploy-data | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 36cbe6780..57f6d93e2 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -27,6 +27,10 @@ if [ -z "$DEPLOY_DATA_YQ_IMAGE" ]; then
     DEPLOY_DATA_YQ_IMAGE="mikefarah/yq:3.3.4"
 fi
 
+if [ -z "$DEPLOY_DATA_RSYNC_IMAGE" ]; then
+    DEPLOY_DATA_RSYNC_IMAGE="eeacms/rsync:2.3"
+fi
+
 CONFIG_YML="$1"
 if [ -z "$CONFIG_YML" ]; then
     echo "ERROR: missing config.yml file" 1>&2
@@ -92,9 +96,21 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
         ensure_not_empty "$DEST_DIR"
 
         echo "sync '$SRC_DIR' to '$DEST_DIR'"
-        rsync --recursive --links --checksum --delete \
-            --itemize-changes --human-readable --verbose \
-            $SRC_DIR $DEST_DIR
+        DEST_DIR_PARENT="`dirname "$DEST_DIR"`"
+        SRC_DIR_ABS_PATH="`pwd`/$SRC_DIR"
+        USER_ID="`id -u`"
+        GROUP_ID="`id -g`"
+
+        # Rsync with --checksum to only update file that changed.
+        docker run --rm --name deploy_data_rsync \
+            --volume $SRC_DIR_ABS_PATH:$SRC_DIR_ABS_PATH:ro \
+            --volume $DEST_DIR_PARENT:$DEST_DIR_PARENT:rw \
+            --user $USER_ID:$GROUP_ID \
+            --entrypoint /usr/bin/rsync \
+            $DEPLOY_DATA_RSYNC_IMAGE \
+                --recursive --links --checksum --delete \
+                --itemize-changes --human-readable --verbose \
+                $SRC_DIR_ABS_PATH $DEST_DIR
 
         DIR_NUM=`expr $DIR_NUM + 1`
     done

From 10485c06e441387a44fe51e6558ea04a03507412 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 7 Oct 2020 19:26:27 -0400
Subject: [PATCH 08/26] deploy-data: avoid name clash with other docker run of
 yq

---
 birdhouse/deployment/deploy-data | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 57f6d93e2..3bd50bcce 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -43,7 +43,7 @@ fi
 
 
 yq() {
-    docker run --rm --name yq -v $CONFIG_YML:$CONFIG_YML:ro $DEPLOY_DATA_YQ_IMAGE yq "$@"
+    docker run --rm --name deploy_data_yq -v $CONFIG_YML:$CONFIG_YML:ro $DEPLOY_DATA_YQ_IMAGE yq "$@"
 }
 
 # Empty value could mean typo in the keys in the config file.

From cf78f3159571bd0386b4daa52f1cc8848e8dbaa0 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 10:28:53 -0400
Subject: [PATCH 09/26] deploy-data: rsync content below source_dir only, not
 including source_dir itself

---
 birdhouse/deployment/deploy-data                   | 2 +-
 birdhouse/deployment/deploy-data.config.sample.yml | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 3bd50bcce..908cbbc3b 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -110,7 +110,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
             $DEPLOY_DATA_RSYNC_IMAGE \
                 --recursive --links --checksum --delete \
                 --itemize-changes --human-readable --verbose \
-                $SRC_DIR_ABS_PATH $DEST_DIR
+                $SRC_DIR_ABS_PATH/ $DEST_DIR
 
         DIR_NUM=`expr $DIR_NUM + 1`
     done
diff --git a/birdhouse/deployment/deploy-data.config.sample.yml b/birdhouse/deployment/deploy-data.config.sample.yml
index 2782ba183..1d47e96f7 100644
--- a/birdhouse/deployment/deploy-data.config.sample.yml
+++ b/birdhouse/deployment/deploy-data.config.sample.yml
@@ -3,27 +3,26 @@
   # branch:
   checkout_name: jenkins-master
   dir_maps:
+  # rsync content below source_dir into dest_dir
   - source_dir: initial-jenkins-plugins-suggestion
     dest_dir: /tmp/deploy-data-test-deploy/jenkins-plugins
 
 - repo_url: https://github.com/Ouranosinc/jenkins-config
-  # optional, default "origin/master"
   branch: origin/master
   checkout_name: jenkins-config
   dir_maps:
-  - source_dir: canarie-presentation
+  - source_dir: canarie-presentation/
     dest_dir: /tmp/deploy-data-test-deploy/canarie
   - source_dir: jcasc
     dest_dir: /tmp/deploy-data-test-deploy/jcasc
 
 - repo_url: https://github.com/Ouranosinc/pavics-sdi
-  # optional, default "origin/master"
   # branch:
   checkout_name: pavics-sdi
   dir_maps:
   - source_dir: docs/source/notebooks
     dest_dir: /tmp/deploy-data-test-deploy/tutorial-notebooks
-  - source_dir: docs/source/processes
+  - source_dir: docs/source/processes/
     dest_dir: /tmp/deploy-data-test-deploy/processes
   - source_dir: docs/source/tutorials
     dest_dir: /tmp/deploy-data-test-deploy/tutorials

From e1e677bdb75c728127aca95cd644a09249064b95 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 10:52:58 -0400
Subject: [PATCH 10/26] deploy-data: ensure rsync have write access to the
 parent folder

---
 birdhouse/deployment/deploy-data | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 908cbbc3b..ae49a5c02 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -101,6 +101,10 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
         USER_ID="`id -u`"
         GROUP_ID="`id -g`"
 
+        # Ensure DEST_DIR_PARENT is created using current USER_ID/GROUP_ID for
+        # next rsync to have proper write access.
+        mkdir -p "$DEST_DIR_PARENT"
+
         # Rsync with --checksum to only update file that changed.
         docker run --rm --name deploy_data_rsync \
             --volume $SRC_DIR_ABS_PATH:$SRC_DIR_ABS_PATH:ro \

From eb95daf7a9cb1e03f5665c05995e824997924226 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 14:50:54 -0400
Subject: [PATCH 11/26] scheduler: new optional cronjob to deploy Raven
 testdata to Thredds

---
 .../deploy_raven_testdata_to_thredds.env      | 46 +++++++++++++++++++
 .../deploy-data-raven-testdata-to-thredds.yml |  8 ++++
 2 files changed, 54 insertions(+)
 create mode 100644 birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
 create mode 100644 birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml

diff --git a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
new file mode 100644
index 000000000..b30f87e5c
--- /dev/null
+++ b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
@@ -0,0 +1,46 @@
+##############################################################################
+# Configuration vars, set in env.local before sourcing this file.
+# This job assume the "scheduler" component is enabled.
+##############################################################################
+
+# Cronjob schedule to trigger deployment attempt.
+if [ -z "$DEPLOY_RAVEN_TESTDATA_SCHEDULE" ]; then
+    DEPLOY_RAVEN_TESTDATA_SCHEDULE="*/30 * * * *"  # UTC
+fi
+
+# Location for local cache of git clone to save bandwidth and time from always
+# re-cloning from scratch.
+if [ -z "$DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE" ]; then
+    DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE="/tmp/deploy_raven_testdata_to_thredds_checkout_cache"
+fi
+
+##############################################################################
+# End configuration vars
+##############################################################################
+
+
+if [ -z "`echo "$AUTODEPLOY_EXTRA_SCHEDULER_JOBS" | grep deploy_raven_testdata_to_thredds`" ]; then
+
+    # Add job only if not already added (config is read twice during
+    # autodeploy process.
+
+    export AUTODEPLOY_EXTRA_SCHEDULER_JOBS="
+$AUTODEPLOY_EXTRA_SCHEDULER_JOBS
+
+- name: deploy_raven_testdata_to_thredds
+  comment: Auto-deploy Raven testdata to Thredds for Raven tutorial notebooks.
+  schedule: '$DEPLOY_RAVEN_TESTDATA_SCHEDULE'
+  command: '/deploy-data /deploy-data-raven-testdata-to-thredds.yml'
+  dockerargs: >-
+    --rm --name deploy_raven_testdata_to_thredds
+    --volume /var/run/docker.sock:/var/run/docker.sock:ro
+    --volume ${COMPOSE_DIR}/deployment/deploy-data:/deploy-data:ro
+    --volume ${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml:/deploy-data-raven-testdata-to-thredds.yml:ro
+    --volume ${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:rw
+    --volume /var/log/PAVICS:/var/log/PAVICS:rw
+    --env DEPLOY_DATA_CHECKOUT_CACHE=${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}
+    --env DEPLOY_DATA_LOGFILE=/var/log/PAVICS/deploy_raven_testdata_to_thredds.log
+  image: 'docker:19.03.6-git'
+"
+
+fi
diff --git a/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml b/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
new file mode 100644
index 000000000..3a3cfdc75
--- /dev/null
+++ b/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
@@ -0,0 +1,8 @@
+- repo_url: https://github.com/Ouranosinc/raven
+  # optional, default "origin/master"
+  # branch:
+  checkout_name: raven
+  dir_maps:
+  # rsync content below source_dir into dest_dir
+  - source_dir: tests/testdata
+    dest_dir: /data/datasets/testdata/raven

From 50a19531f0dbad656b3a16dd9e77d3929b59b85a Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 15:19:03 -0400
Subject: [PATCH 12/26] deploy raven testdata cronjob: yml config file path has
 to exist on host due to volume mount

Fix following error:

+ docker run --rm --name deploy_data_yq -v /deploy-data-raven-testdata-to-thredds.yml:/deploy-data-raven-testdata-to-thredds.yml:ro mikefarah/yq:3.3.4 yq r -p v /deploy-data-raven-testdata-to-thredds.yml '[*].repo_url'
Error: yaml: input error: read /deploy-data-raven-testdata-to-thredds.yml: is a directory
---
 .../components/scheduler/deploy_raven_testdata_to_thredds.env | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
index b30f87e5c..083def8fa 100644
--- a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
+++ b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
@@ -30,12 +30,12 @@ $AUTODEPLOY_EXTRA_SCHEDULER_JOBS
 - name: deploy_raven_testdata_to_thredds
   comment: Auto-deploy Raven testdata to Thredds for Raven tutorial notebooks.
   schedule: '$DEPLOY_RAVEN_TESTDATA_SCHEDULE'
-  command: '/deploy-data /deploy-data-raven-testdata-to-thredds.yml'
+  command: '/deploy-data ${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml'
   dockerargs: >-
     --rm --name deploy_raven_testdata_to_thredds
     --volume /var/run/docker.sock:/var/run/docker.sock:ro
     --volume ${COMPOSE_DIR}/deployment/deploy-data:/deploy-data:ro
-    --volume ${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml:/deploy-data-raven-testdata-to-thredds.yml:ro
+    --volume ${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml:${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml:ro
     --volume ${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:rw
     --volume /var/log/PAVICS:/var/log/PAVICS:rw
     --env DEPLOY_DATA_CHECKOUT_CACHE=${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}

From 4e73a85912293da3baefd86df83c9e98de0e5bb2 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 17:43:31 -0400
Subject: [PATCH 13/26] deploy-data: expose rsync options for include/exlude
 filter rules

Raven testdata sync log:

+ docker run --rm --name deploy_data_rsync --volume /tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven/tests/testdata:/tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven/tests/testdata:ro --volume /data/datasets/testdata:/data/datasets/testdata:rw --user 0:0 --entrypoint /usr/bin/rsync eeacms/rsync:2.3 --recursive --links --checksum --delete --itemize-changes --human-readable --verbose --prune-empty-dirs '--include=*/' '--include=*.nc' '--exclude=*' /tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven/tests/testdata/ /data/datasets/testdata/raven
building file list ... done
created directory /data/datasets/testdata/raven
cd+++++++++ ./
cd+++++++++ XSS_forecast_data/
>f+++++++++ XSS_forecast_data/XSS_fcst_det.nc
>f+++++++++ XSS_forecast_data/XSS_fcst_ens.nc
>f+++++++++ XSS_forecast_data/XSS_obs.nc
cd+++++++++ cmip5/
>f+++++++++ cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200601-210012_subset.nc
cd+++++++++ gr4j_cemaneige/
>f+++++++++ gr4j_cemaneige/evap.nc
>f+++++++++ gr4j_cemaneige/pr.nc
>f+++++++++ gr4j_cemaneige/tas.nc
cd+++++++++ hydro_simulations/
>f+++++++++ hydro_simulations/raven-gr4j-cemaneige-sim_gr4jcn-0_Hydrographs.nc
>f+++++++++ hydro_simulations/raven-gr4j-cemaneige-sim_hmets-0_Hydrographs.nc
cd+++++++++ input2d/
>f+++++++++ input2d/input2d.nc
cd+++++++++ ostrich-gr4j-cemaneige/
>f+++++++++ ostrich-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc
cd+++++++++ ostrich-hbv-ec/
>f+++++++++ ostrich-hbv-ec/Salmon-River-Near-Prince-George_meteo_daily.nc
cd+++++++++ ostrich-hmets/
>f+++++++++ ostrich-hmets/Salmon-River-Near-Prince-George_meteo_daily.nc
cd+++++++++ ostrich-mohyse/
>f+++++++++ ostrich-mohyse/Salmon-River-Near-Prince-George_meteo_daily.nc
cd+++++++++ raven-gr4j-cemaneige/
>f+++++++++ raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc
>f+++++++++ raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily_2d.nc
>f+++++++++ raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily_3d.nc
cd+++++++++ ts_stats_outputs/
>f+++++++++ ts_stats_outputs/out.nc

sent 18.72M bytes  received 446 bytes  37.44M bytes/sec
total size is 18.71M  speedup is 1.00

Test delete, modify, add new .nc file:

+ docker run --rm --name deploy_data_rsync --volume /tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven/tests/testdata:/tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven/tests/testdata:ro --volume /data/datasets/testdata:/data/datasets/testdata:rw --user 0:0 --entrypoint /usr/bin/rsync eeacms/rsync:2.3 --recursive --links --checksum --delete --itemize-changes --human-readable --verbose --prune-empty-dirs '--include=*/' '--include=*.nc' '--exclude=*' /tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven/tests/testdata/ /data/datasets/testdata/raven
building file list ... done
*deleting   gr4j_cemaneige/evap.nc
>fcsT...... gr4j_cemaneige/pr.nc
>f+++++++++ gr4j_cemaneige/toto.nc
---
 birdhouse/deployment/deploy-data                           | 2 ++
 .../deployment/deploy-data-raven-testdata-to-thredds.yml   | 1 +
 birdhouse/deployment/deploy-data.config.sample.yml         | 7 +++++++
 3 files changed, 10 insertions(+)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index ae49a5c02..266e64520 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -94,6 +94,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
     for SRC_DIR in $SRC_DIRS; do
         DEST_DIR="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[$DIR_NUM\].dest_dir`"
         ensure_not_empty "$DEST_DIR"
+        RSYNC_EXTRA_OPTS="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[$DIR_NUM\].rsync_extra_opts`"
 
         echo "sync '$SRC_DIR' to '$DEST_DIR'"
         DEST_DIR_PARENT="`dirname "$DEST_DIR"`"
@@ -114,6 +115,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
             $DEPLOY_DATA_RSYNC_IMAGE \
                 --recursive --links --checksum --delete \
                 --itemize-changes --human-readable --verbose \
+                --prune-empty-dirs $RSYNC_EXTRA_OPTS \
                 $SRC_DIR_ABS_PATH/ $DEST_DIR
 
         DIR_NUM=`expr $DIR_NUM + 1`
diff --git a/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml b/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
index 3a3cfdc75..f085c8e14 100644
--- a/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
+++ b/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
@@ -6,3 +6,4 @@
   # rsync content below source_dir into dest_dir
   - source_dir: tests/testdata
     dest_dir: /data/datasets/testdata/raven
+    rsync_extra_opts: --include=*/ --include=*.nc --exclude=*
diff --git a/birdhouse/deployment/deploy-data.config.sample.yml b/birdhouse/deployment/deploy-data.config.sample.yml
index 1d47e96f7..4cc6f5fca 100644
--- a/birdhouse/deployment/deploy-data.config.sample.yml
+++ b/birdhouse/deployment/deploy-data.config.sample.yml
@@ -6,6 +6,8 @@
   # rsync content below source_dir into dest_dir
   - source_dir: initial-jenkins-plugins-suggestion
     dest_dir: /tmp/deploy-data-test-deploy/jenkins-plugins
+    # optional, useful for include/exclude filter rules
+    # rsync_extra_opts:
 
 - repo_url: https://github.com/Ouranosinc/jenkins-config
   branch: origin/master
@@ -13,8 +15,10 @@
   dir_maps:
   - source_dir: canarie-presentation/
     dest_dir: /tmp/deploy-data-test-deploy/canarie
+    rsync_extra_opts:
   - source_dir: jcasc
     dest_dir: /tmp/deploy-data-test-deploy/jcasc
+    rsync_extra_opts:
 
 - repo_url: https://github.com/Ouranosinc/pavics-sdi
   # branch:
@@ -22,7 +26,10 @@
   dir_maps:
   - source_dir: docs/source/notebooks
     dest_dir: /tmp/deploy-data-test-deploy/tutorial-notebooks
+    # rsync_extra_opts:
   - source_dir: docs/source/processes/
     dest_dir: /tmp/deploy-data-test-deploy/processes
+    # rsync_extra_opts:
   - source_dir: docs/source/tutorials
     dest_dir: /tmp/deploy-data-test-deploy/tutorials
+    # rsync_extra_opts:

From 0f23b8c4bc88bad2781c96b7b99c19ed351421ef Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 18:00:55 -0400
Subject: [PATCH 14/26] deploy raven testdata cronjob: allow to customize log
 file name and location

---
 .../scheduler/deploy_raven_testdata_to_thredds.env    | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
index 083def8fa..6a2932846 100644
--- a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
+++ b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
@@ -14,6 +14,11 @@ if [ -z "$DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE" ]; then
     DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE="/tmp/deploy_raven_testdata_to_thredds_checkout_cache"
 fi
 
+# Log file location.  Default location under /var/log/PAVICS/ has built-in logrotate.
+if [ -z "$DEPLOY_RAVEN_TESTDATA_LOGFILE" ]; then
+    DEPLOY_RAVEN_TESTDATA_LOGFILE="/var/log/PAVICS/deploy_raven_testdata_to_thredds.log"
+fi
+
 ##############################################################################
 # End configuration vars
 ##############################################################################
@@ -24,6 +29,8 @@ if [ -z "`echo "$AUTODEPLOY_EXTRA_SCHEDULER_JOBS" | grep deploy_raven_testdata_t
     # Add job only if not already added (config is read twice during
     # autodeploy process.
 
+    LOGFILE_DIRNAME="`dirname "$DEPLOY_RAVEN_TESTDATA_LOGFILE"`"
+
     export AUTODEPLOY_EXTRA_SCHEDULER_JOBS="
 $AUTODEPLOY_EXTRA_SCHEDULER_JOBS
 
@@ -37,9 +44,9 @@ $AUTODEPLOY_EXTRA_SCHEDULER_JOBS
     --volume ${COMPOSE_DIR}/deployment/deploy-data:/deploy-data:ro
     --volume ${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml:${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml:ro
     --volume ${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:rw
-    --volume /var/log/PAVICS:/var/log/PAVICS:rw
+    --volume ${LOGFILE_DIRNAME}:${LOGFILE_DIRNAME}:rw
     --env DEPLOY_DATA_CHECKOUT_CACHE=${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}
-    --env DEPLOY_DATA_LOGFILE=/var/log/PAVICS/deploy_raven_testdata_to_thredds.log
+    --env DEPLOY_DATA_LOGFILE=${DEPLOY_RAVEN_TESTDATA_LOGFILE}
   image: 'docker:19.03.6-git'
 "
 

From fe41c84a68c9aad56522e67fde64ee17f63ccbbb Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 20:27:01 -0400
Subject: [PATCH 15/26] deploy-data: add documentation

---
 birdhouse/deployment/deploy-data              | 20 ++++++++++++++++++-
 .../deployment/deploy-data.config.sample.yml  |  6 ++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 266e64520..d1d2eb002 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -1,7 +1,25 @@
 #!/bin/sh
 # Deploy data from git repo(s) to local folder(s).
 #
-# See sample input config in deploy-data.config.sample.yml.
+# See sample input config in deploy-data.config.sample.yml for how to specify
+# which git repo(s), which git branch for each repo, which sub-folder(s) to
+# sync to which local folder(s) and rsync extra options for each sub-folder.
+#
+# The git repo clones are cached for faster subsequent runs and rsync is used
+# to only modify files that actually changed, to keep the file tree in sync and
+# to have include/exclude filter rules.  All these options are not available if
+# using regular 'cp'.
+#
+# Docker image is used for yq (yaml file parser) and rsync so this script have
+# very few install dependencies (only need docker and git installed locally)
+# so it can runs inside very minimalistic image (the 'docker' Docker image).
+#
+# Setting environment variable DEPLOY_DATA_LOGFILE='/path/to/logfile.log'
+# will redirect all STDOUT and STDERR to that logfile so this script will be
+# completely silent.
+#
+# Other self explanatory environment variables DEPLOY_DATA_CHECKOUT_CACHE,
+# DEPLOY_DATA_YQ_IMAGE, DEPLOY_DATA_RSYNC_IMAGE.
 #
 
 if [ ! -z "$DEPLOY_DATA_LOGFILE" ]; then
diff --git a/birdhouse/deployment/deploy-data.config.sample.yml b/birdhouse/deployment/deploy-data.config.sample.yml
index 4cc6f5fca..0bfcd9dd1 100644
--- a/birdhouse/deployment/deploy-data.config.sample.yml
+++ b/birdhouse/deployment/deploy-data.config.sample.yml
@@ -1,3 +1,9 @@
+# Sample config file for deploy-data script.
+#
+# Many git repos are supported.  For each repo, many mapping between source dir
+# and destination dir are supported.  For each mapping, extra rsync option can
+# be provided to include/exclude a subset of files to keep in sync.
+
 - repo_url: https://github.com/Ouranosinc/jenkins-master
   # optional, default "origin/master"
   # branch:

From 9d3dd19a339978a3d829c35eae8872cb11531415 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 20:36:30 -0400
Subject: [PATCH 16/26] env.local: reference optional cronjob to auto deploy
 Raven testdata to Thredds

---
 birdhouse/env.local.example                 | 12 ++++++++++++
 birdhouse/vagrant-utils/configure-pavics.sh |  6 +++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
index 4c50fdd2a..57cc0e624 100644
--- a/birdhouse/env.local.example
+++ b/birdhouse/env.local.example
@@ -147,7 +147,19 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
 # See the job for additional possible configurations.  The "scheduler"
 # component needs to be enabled for this pre-configured job to work.
 #
+#if [ -f "/<absolute path>/components/scheduler/renew_letsencrypt_ssl_cert_extra_job.env" ]; then
 #. /<absolute path>/components/scheduler/renew_letsencrypt_ssl_cert_extra_job.env
+#fi
+#
+# Load pre-configured cronjob to automatically deploy Raven testdata to Thredds
+# for Raven tutorial notebooks.
+#
+# See the job for additional possible configurations.  The "scheduler"
+# component needs to be enabled for this pre-configured job to work.
+#
+#if [ -f "/<absolute path>/components/scheduler/deploy_raven_testdata_to_thredds.env" ]; then
+#. /<absolute path>/components/scheduler/deploy_raven_testdata_to_thredds.env
+#fi
 
 # Public (on the internet) fully qualified domain name of this Pavics
 # installation.  This is optional so default to the same internal PAVICS_FQDN if
diff --git a/birdhouse/vagrant-utils/configure-pavics.sh b/birdhouse/vagrant-utils/configure-pavics.sh
index ef738fc4a..533d7631b 100755
--- a/birdhouse/vagrant-utils/configure-pavics.sh
+++ b/birdhouse/vagrant-utils/configure-pavics.sh
@@ -30,7 +30,11 @@ RENEW_LETSENCRYPT_SSL_SCHEDULE="22 9 * * *"  # UTC
 # This repo will be volume-mount at /vagrant so can not go higher.
 RENEW_LETSENCRYPT_SSL_NUM_PARENTS_MOUNT="/"
 
-. $PWD/components/scheduler/renew_letsencrypt_ssl_cert_extra_job.env
+# Only source if file exist.  Allow for config file to be backward-compat with
+# older version of the repo where the .env file do not exist yet.
+if [ -f "$PWD/components/scheduler/renew_letsencrypt_ssl_cert_extra_job.env" ]; then
+    . $PWD/components/scheduler/renew_letsencrypt_ssl_cert_extra_job.env
+fi
 EOF
     elif [ -n "$KITENAME" -a -n "$KITESUBDOMAIN" ]; then
     cat <<EOF >> env.local

From 89361990142a33985333b2d9e7b12ee0051c6f41 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Thu, 8 Oct 2020 20:47:23 -0400
Subject: [PATCH 17/26] README: reference Ouranos specific override repo to
 demo infra-as-code

---
 birdhouse/README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/birdhouse/README.md b/birdhouse/README.md
index 8241bb0ae..0c6bf07cc 100644
--- a/birdhouse/README.md
+++ b/birdhouse/README.md
@@ -25,7 +25,10 @@ for your organization.  For an example of possible override, see how the [emu
 service](optional-components/emu/docker-compose-extra.yml)
 ([README](optional-components/README.md)) can be optionally added to the
 deployment via the [override
-mechanism](https://docs.docker.com/compose/extends/).
+mechanism](https://docs.docker.com/compose/extends/).  Ouranos specific
+override can be found in this
+[birdhouse-deploy-ouranos](https://github.com/bird-house/birdhouse-deploy-ouranos)
+repo.
 
 The automatic deployment is able to handle multiple repos, so will trigger if
 this repo or your private-personalized-config repo changes, giving you

From 259f228bc18699e7c060f1fb426b2181b37ae3a0 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 16:21:07 -0400
Subject: [PATCH 18/26] deploy-data: add more include and very basic remap
 example

---
 .../deployment/deploy-data.config.sample.yml  | 29 ++++++++++++-------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/birdhouse/deployment/deploy-data.config.sample.yml b/birdhouse/deployment/deploy-data.config.sample.yml
index 0bfcd9dd1..f83d1de98 100644
--- a/birdhouse/deployment/deploy-data.config.sample.yml
+++ b/birdhouse/deployment/deploy-data.config.sample.yml
@@ -21,21 +21,28 @@
   dir_maps:
   - source_dir: canarie-presentation/
     dest_dir: /tmp/deploy-data-test-deploy/canarie
-    rsync_extra_opts:
+    # sync only .txt, .html and .gif files, if other already existing files,
+    # ignore them, unless they have same extensions.
+    rsync_extra_opts: --include=*/ --include=*.txt --include=*.html --include=*.gif --exclude=*
   - source_dir: jcasc
-    dest_dir: /tmp/deploy-data-test-deploy/jcasc
+    # remap dir jcasc inside previous dir canarie, without conflicting with
+    # previous canarie sync.  This works because no .txt, .html, .gif in jcasc.
+    dest_dir: /tmp/deploy-data-test-deploy/canarie/jcasc
     rsync_extra_opts:
 
 - repo_url: https://github.com/Ouranosinc/pavics-sdi
   # branch:
   checkout_name: pavics-sdi
   dir_maps:
-  - source_dir: docs/source/notebooks
-    dest_dir: /tmp/deploy-data-test-deploy/tutorial-notebooks
-    # rsync_extra_opts:
-  - source_dir: docs/source/processes/
-    dest_dir: /tmp/deploy-data-test-deploy/processes
-    # rsync_extra_opts:
-  - source_dir: docs/source/tutorials
-    dest_dir: /tmp/deploy-data-test-deploy/tutorials
-    # rsync_extra_opts:
+    # sync only 2 sub-dirs and .rst files under source/
+  - source_dir: docs/
+    dest_dir: /tmp/deploy-data-test-deploy/pavics-sdi
+    rsync_extra_opts: --include=*/ --include=source/tutorials/** --include=source/processes/** --include=source/*.rst --exclude=*
+    # sync only .yml files at the root of checkout
+  - source_dir: .
+    dest_dir: /tmp/deploy-data-test-deploy/pavics-sdi
+    rsync_extra_opts: --include=/ --include=*.yml --exclude=*
+    # move dir 'notebooks' one level higher in hierarchy
+  - source_dir: docs/source
+    dest_dir: /tmp/deploy-data-test-deploy/pavics-sdi
+    rsync_extra_opts: --include=*/ --include=notebooks/** --exclude=*

From a3937a4ba6db054e82c04fa3dd253410bb907b8b Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 16:41:50 -0400
Subject: [PATCH 19/26] deploy-data: allow checkout cache config to be set in
 config file

Other configs need to be already set before config file parsing occur so
can not be set in config file.
---
 birdhouse/deployment/deploy-data               | 18 +++++++++++-------
 .../deploy-data-raven-testdata-to-thredds.yml  |  2 ++
 .../deployment/deploy-data.config.sample.yml   |  7 +++++++
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index d1d2eb002..456b1d232 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -79,18 +79,22 @@ datadeploy START_TIME=$START_TIME"
 
 set -x
 
-GIT_REPO_URLS="`yq r -p v $CONFIG_YML \[*\].repo_url`"
+GIT_REPO_URLS="`yq r -p v $CONFIG_YML deploy\[*\].repo_url`"
 ensure_not_empty "$GIT_REPO_URLS"
 REPO_NUM=0
 
 for GIT_REPO_URL in $GIT_REPO_URLS; do
 
-    GIT_BRANCH="`yq r -p v $CONFIG_YML --defaultValue origin/master \[$REPO_NUM\].branch`"
+    GIT_BRANCH="`yq r -p v $CONFIG_YML --defaultValue origin/master deploy\[$REPO_NUM\].branch`"
     ensure_not_empty "$GIT_BRANCH"
-    GIT_CHECKOUT_NAME="`yq r -p v $CONFIG_YML \[$REPO_NUM\].checkout_name`"
+    GIT_CHECKOUT_NAME="`yq r -p v $CONFIG_YML deploy\[$REPO_NUM\].checkout_name`"
     ensure_not_empty "$GIT_CHECKOUT_NAME"
 
-    CLONE_DEST="$DEPLOY_DATA_CHECKOUT_CACHE/$GIT_CHECKOUT_NAME"
+    CHECKOUT_CACHE="`yq r -p v $CONFIG_YML config.checkout_cache`"
+    if [ -z "$CHECKOUT_CACHE" ]; then
+        CHECKOUT_CACHE="$DEPLOY_DATA_CHECKOUT_CACHE"
+    fi
+    CLONE_DEST="$CHECKOUT_CACHE/$GIT_CHECKOUT_NAME"
     if [ ! -d "$CLONE_DEST" ]; then
         echo "checkout repo '$GIT_REPO_URL' on branch '$GIT_BRANCH' to '$CLONE_DEST'"
         git clone $GIT_REPO_URL $CLONE_DEST
@@ -105,14 +109,14 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
         git checkout --force $GIT_BRANCH  # force checkout to throwaway local changes
     fi
 
-    SRC_DIRS="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[*\].source_dir`"
+    SRC_DIRS="`yq r -p v $CONFIG_YML deploy\[$REPO_NUM\].dir_maps\[*\].source_dir`"
     ensure_not_empty "$SRC_DIRS"
     DIR_NUM=0
 
     for SRC_DIR in $SRC_DIRS; do
-        DEST_DIR="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[$DIR_NUM\].dest_dir`"
+        DEST_DIR="`yq r -p v $CONFIG_YML deploy\[$REPO_NUM\].dir_maps\[$DIR_NUM\].dest_dir`"
         ensure_not_empty "$DEST_DIR"
-        RSYNC_EXTRA_OPTS="`yq r -p v $CONFIG_YML \[$REPO_NUM\].dir_maps\[$DIR_NUM\].rsync_extra_opts`"
+        RSYNC_EXTRA_OPTS="`yq r -p v $CONFIG_YML deploy\[$REPO_NUM\].dir_maps\[$DIR_NUM\].rsync_extra_opts`"
 
         echo "sync '$SRC_DIR' to '$DEST_DIR'"
         DEST_DIR_PARENT="`dirname "$DEST_DIR"`"
diff --git a/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml b/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
index f085c8e14..a021ea986 100644
--- a/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
+++ b/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
@@ -1,3 +1,4 @@
+deploy:
 - repo_url: https://github.com/Ouranosinc/raven
   # optional, default "origin/master"
   # branch:
@@ -6,4 +7,5 @@
   # rsync content below source_dir into dest_dir
   - source_dir: tests/testdata
     dest_dir: /data/datasets/testdata/raven
+    # only sync .nc files
     rsync_extra_opts: --include=*/ --include=*.nc --exclude=*
diff --git a/birdhouse/deployment/deploy-data.config.sample.yml b/birdhouse/deployment/deploy-data.config.sample.yml
index f83d1de98..7e654f9b7 100644
--- a/birdhouse/deployment/deploy-data.config.sample.yml
+++ b/birdhouse/deployment/deploy-data.config.sample.yml
@@ -4,6 +4,13 @@
 # and destination dir are supported.  For each mapping, extra rsync option can
 # be provided to include/exclude a subset of files to keep in sync.
 
+config:
+  # optional, default "/tmp/deploy-data-clone-cache"
+  # can also be set by env var DEPLOY_DATA_CHECKOUT_CACHE
+  # setting in this config file have precedence over env var
+  # checkout_cache:
+
+deploy:
 - repo_url: https://github.com/Ouranosinc/jenkins-master
   # optional, default "origin/master"
   # branch:

From f80a614c3674ce66497c40ee2cfb0eb0eba923b5 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 17:57:36 -0400
Subject: [PATCH 20/26] deploy raven testdata cronjob: allow to provide
 alternate config file

---
 .../scheduler/deploy_raven_testdata_to_thredds.env    | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
index 6a2932846..97b3c3236 100644
--- a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
+++ b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
@@ -14,6 +14,13 @@ if [ -z "$DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE" ]; then
     DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE="/tmp/deploy_raven_testdata_to_thredds_checkout_cache"
 fi
 
+# Location of deploy-data config file.
+# Provide a different config file to sync to a different location or include
+# more files in the sync.
+if [ -z "$DEPLOY_RAVEN_TESTDATA_CONFIG" ]; then
+    DEPLOY_RAVEN_TESTDATA_CONFIG="${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml"
+fi
+
 # Log file location.  Default location under /var/log/PAVICS/ has built-in logrotate.
 if [ -z "$DEPLOY_RAVEN_TESTDATA_LOGFILE" ]; then
     DEPLOY_RAVEN_TESTDATA_LOGFILE="/var/log/PAVICS/deploy_raven_testdata_to_thredds.log"
@@ -37,12 +44,12 @@ $AUTODEPLOY_EXTRA_SCHEDULER_JOBS
 - name: deploy_raven_testdata_to_thredds
   comment: Auto-deploy Raven testdata to Thredds for Raven tutorial notebooks.
   schedule: '$DEPLOY_RAVEN_TESTDATA_SCHEDULE'
-  command: '/deploy-data ${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml'
+  command: '/deploy-data ${DEPLOY_RAVEN_TESTDATA_CONFIG}'
   dockerargs: >-
     --rm --name deploy_raven_testdata_to_thredds
     --volume /var/run/docker.sock:/var/run/docker.sock:ro
     --volume ${COMPOSE_DIR}/deployment/deploy-data:/deploy-data:ro
-    --volume ${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml:${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml:ro
+    --volume ${DEPLOY_RAVEN_TESTDATA_CONFIG}:${DEPLOY_RAVEN_TESTDATA_CONFIG}:ro
     --volume ${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:rw
     --volume ${LOGFILE_DIRNAME}:${LOGFILE_DIRNAME}:rw
     --env DEPLOY_DATA_CHECKOUT_CACHE=${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}

From 8d765261dd35751b935a577cca6d8e9042c35ffe Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 19:07:48 -0400
Subject: [PATCH 21/26] deploy-data: support git clone over ssh for private
 repos

---
 birdhouse/deployment/deploy-data              | 21 +++++++++++++++----
 .../deployment/deploy-data.config.sample.yml  |  9 +++++++-
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 456b1d232..3cca1da80 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -18,6 +18,9 @@
 # will redirect all STDOUT and STDERR to that logfile so this script will be
 # completely silent.
 #
+# Setting environment variable DEPLOY_DATA_GIT_SSH_IDENTITY_FILE='/path/to/id_rsa'
+# will allow git clone over ssh, useful for private repos.
+#
 # Other self explanatory environment variables DEPLOY_DATA_CHECKOUT_CACHE,
 # DEPLOY_DATA_YQ_IMAGE, DEPLOY_DATA_RSYNC_IMAGE.
 #
@@ -79,6 +82,20 @@ datadeploy START_TIME=$START_TIME"
 
 set -x
 
+CHECKOUT_CACHE="`yq r -p v $CONFIG_YML config.checkout_cache`"
+if [ -z "$CHECKOUT_CACHE" ]; then
+    CHECKOUT_CACHE="$DEPLOY_DATA_CHECKOUT_CACHE"
+fi
+
+GIT_SSH_IDENTITY_FILE="`yq r -p v $CONFIG_YML config.git_ssh_identity_file`"
+if [ -z "$GIT_SSH_IDENTITY_FILE" ]; then
+    GIT_SSH_IDENTITY_FILE="$DEPLOY_DATA_GIT_SSH_IDENTITY_FILE"
+fi
+
+if [ ! -z "$GIT_SSH_IDENTITY_FILE" ]; then
+    export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=$GIT_SSH_IDENTITY_FILE"
+fi
+
 GIT_REPO_URLS="`yq r -p v $CONFIG_YML deploy\[*\].repo_url`"
 ensure_not_empty "$GIT_REPO_URLS"
 REPO_NUM=0
@@ -90,10 +107,6 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
     GIT_CHECKOUT_NAME="`yq r -p v $CONFIG_YML deploy\[$REPO_NUM\].checkout_name`"
     ensure_not_empty "$GIT_CHECKOUT_NAME"
 
-    CHECKOUT_CACHE="`yq r -p v $CONFIG_YML config.checkout_cache`"
-    if [ -z "$CHECKOUT_CACHE" ]; then
-        CHECKOUT_CACHE="$DEPLOY_DATA_CHECKOUT_CACHE"
-    fi
     CLONE_DEST="$CHECKOUT_CACHE/$GIT_CHECKOUT_NAME"
     if [ ! -d "$CLONE_DEST" ]; then
         echo "checkout repo '$GIT_REPO_URL' on branch '$GIT_BRANCH' to '$CLONE_DEST'"
diff --git a/birdhouse/deployment/deploy-data.config.sample.yml b/birdhouse/deployment/deploy-data.config.sample.yml
index 7e654f9b7..0552b09c6 100644
--- a/birdhouse/deployment/deploy-data.config.sample.yml
+++ b/birdhouse/deployment/deploy-data.config.sample.yml
@@ -8,9 +8,16 @@ config:
   # optional, default "/tmp/deploy-data-clone-cache"
   # can also be set by env var DEPLOY_DATA_CHECKOUT_CACHE
   # setting in this config file have precedence over env var
-  # checkout_cache:
+  #checkout_cache:
+
+  # optional, default unset
+  # for git clone over ssh, useful for private repos
+  # can also be set by env var DEPLOY_DATA_GIT_SSH_IDENTITY_FILE
+  # setting in this config file have precedence over env var
+  #git_ssh_identity_file: /path/to/id_rsa
 
 deploy:
+  # this form if clone over ssh: git@github.com:Ouranosinc/jenkins-master.git
 - repo_url: https://github.com/Ouranosinc/jenkins-master
   # optional, default "origin/master"
   # branch:

From 91a583b4ed05c85356c230cd632da1cd8f98b3ad Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 20:03:06 -0400
Subject: [PATCH 22/26] deploy raven testdata cronjob: support git clone over
 ssh for private repos

+ docker run --rm --name deploy_data_yq -v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml:/vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml:ro mikefarah/yq:3.3.4 yq r -p v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml config.git_ssh_identity_file
+ GIT_SSH_IDENTITY_FILE=
+ '[' -z  ]
+ GIT_SSH_IDENTITY_FILE=/home/vagrant/.ssh/id_rsa_git_ssh_read_only
+ '[' '!' -z /home/vagrant/.ssh/id_rsa_git_ssh_read_only ]
+ export 'GIT_SSH_COMMAND=ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=/home/vagrant/.ssh/id_rsa_git_ssh_read_only'
+ yq r -p v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml 'deploy[*].repo_url'
+ docker run --rm --name deploy_data_yq -v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml:/vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml:ro mikefarah/yq:3.3.4 yq r -p v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml 'deploy[*].repo_url'
+ GIT_REPO_URLS=git@github.com:Ouranosinc/raven.git
+ ensure_not_empty git@github.com:Ouranosinc/raven.git
+ '[' -z git@github.com:Ouranosinc/raven.git ]
+ REPO_NUM=0
+ yq r -p v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml --defaultValue origin/master 'deploy[0].branch'
+ docker run --rm --name deploy_data_yq -v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml:/vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml:ro mikefarah/yq:3.3.4 yq r -p v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml --defaultValue origin/master 'deploy[0].branch'
+ GIT_BRANCH=origin/master
+ ensure_not_empty origin/master
+ '[' -z origin/master ]
+ yq r -p v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml 'deploy[0].checkout_name'
+ docker run --rm --name deploy_data_yq -v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml:/vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml:ro mikefarah/yq:3.3.4 yq r -p v /vagrant/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml 'deploy[0].checkout_name'
+ GIT_CHECKOUT_NAME=raven
+ ensure_not_empty raven
+ '[' -z raven ]
+ CLONE_DEST=/tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven
+ '[' '!' -d /tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven ]
+ echo 'checkout repo '"'"'git@github.com:Ouranosinc/raven.git'"'"' on branch '"'"'origin/master'"'"' to '"'"'/tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven'"'"
checkout repo 'git@github.com:Ouranosinc/raven.git' on branch 'origin/master' to '/tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven'
+ git clone git@github.com:Ouranosinc/raven.git /tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven
Cloning into '/tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven'...
Warning: Permanently added 'github.com,140.82.114.4' (RSA) to the list of known hosts.^M
Updating files:  99% (382/383)^MUpdating files: 100% (383/383)^MUpdating files: 100% (383/383), done.
+ cd /tmp/deploy_raven_testdata_to_thredds_checkout_cache/raven
+ git checkout origin/master
Note: switching to 'origin/master'.

You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by switching back to a branch.

If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -c with the switch command. Example:

  git switch -c <new-branch-name>

Or undo this operation with:

  git switch -

Turn off this advice by setting config variable advice.detachedHead to false

HEAD is now at 58e8978 Merge pull request #279 from Ouranosinc/climatologyESP
---
 .../deploy_raven_testdata_to_thredds.env          | 15 ++++++++++++++-
 .../deploy-data-raven-testdata-to-thredds.yml     |  1 +
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
index 97b3c3236..574f7a46e 100644
--- a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
+++ b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
@@ -26,6 +26,12 @@ if [ -z "$DEPLOY_RAVEN_TESTDATA_LOGFILE" ]; then
     DEPLOY_RAVEN_TESTDATA_LOGFILE="/var/log/PAVICS/deploy_raven_testdata_to_thredds.log"
 fi
 
+# Location of ssh private key for git clone over ssh, useful for private repos.
+# Raven do not need this since Raven repo is public so cloning over https.
+# This is here in case a custom config file is supplied with additional repos.
+#DEPLOY_RAVEN_TESTDATA_GIT_SSH_IDENTITY_FILE="/path/to/id_rsa"
+#DEPLOY_RAVEN_TESTDATA_GIT_SSH_IDENTITY_FILE=/home/vagrant/.ssh/id_rsa_git_ssh_read_only
+
 ##############################################################################
 # End configuration vars
 ##############################################################################
@@ -38,6 +44,13 @@ if [ -z "`echo "$AUTODEPLOY_EXTRA_SCHEDULER_JOBS" | grep deploy_raven_testdata_t
 
     LOGFILE_DIRNAME="`dirname "$DEPLOY_RAVEN_TESTDATA_LOGFILE"`"
 
+    EXTRA_DOCKER_ARGS=""
+    if [ -n "$DEPLOY_RAVEN_TESTDATA_GIT_SSH_IDENTITY_FILE" ]; then
+        EXTRA_DOCKER_ARGS="
+    --volume ${DEPLOY_RAVEN_TESTDATA_GIT_SSH_IDENTITY_FILE}:${DEPLOY_RAVEN_TESTDATA_GIT_SSH_IDENTITY_FILE}:ro
+    --env DEPLOY_DATA_GIT_SSH_IDENTITY_FILE=${DEPLOY_RAVEN_TESTDATA_GIT_SSH_IDENTITY_FILE}"
+    fi
+
     export AUTODEPLOY_EXTRA_SCHEDULER_JOBS="
 $AUTODEPLOY_EXTRA_SCHEDULER_JOBS
 
@@ -53,7 +66,7 @@ $AUTODEPLOY_EXTRA_SCHEDULER_JOBS
     --volume ${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}:rw
     --volume ${LOGFILE_DIRNAME}:${LOGFILE_DIRNAME}:rw
     --env DEPLOY_DATA_CHECKOUT_CACHE=${DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE}
-    --env DEPLOY_DATA_LOGFILE=${DEPLOY_RAVEN_TESTDATA_LOGFILE}
+    --env DEPLOY_DATA_LOGFILE=${DEPLOY_RAVEN_TESTDATA_LOGFILE} ${EXTRA_DOCKER_ARGS}
   image: 'docker:19.03.6-git'
 "
 
diff --git a/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml b/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
index a021ea986..30e2bc9b1 100644
--- a/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
+++ b/birdhouse/deployment/deploy-data-raven-testdata-to-thredds.yml
@@ -1,4 +1,5 @@
 deploy:
+#- repo_url: git@github.com:Ouranosinc/raven.git
 - repo_url: https://github.com/Ouranosinc/raven
   # optional, default "origin/master"
   # branch:

From 5e79750c974e9a1d65cd2515bcb782cd19c3f5c9 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 20:16:13 -0400
Subject: [PATCH 23/26] deploy-data: additional error checking for git clone
 and fetch that require credentials

+ git fetch --prune --all
Fetching origin
Host key verification failed.^M
fatal: Could not read from remote repository.

Please make sure you have the correct access rights
and the repository exists.
error: Could not fetch origin
+ echo 'git fetch failed'
git fetch failed
+ exit 1
---
 birdhouse/deployment/deploy-data | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 3cca1da80..4da17c8fc 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -110,7 +110,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
     CLONE_DEST="$CHECKOUT_CACHE/$GIT_CHECKOUT_NAME"
     if [ ! -d "$CLONE_DEST" ]; then
         echo "checkout repo '$GIT_REPO_URL' on branch '$GIT_BRANCH' to '$CLONE_DEST'"
-        git clone $GIT_REPO_URL $CLONE_DEST
+        git clone $GIT_REPO_URL $CLONE_DEST || echo "git clone failed" 1>&2 ; exit 1
         cd $CLONE_DEST
         git checkout $GIT_BRANCH
     else
@@ -118,7 +118,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
         cd $CLONE_DEST
         git remote -v  # log remote, should match GIT_REPO_URL
         git clean -fdx  # force, recur dir, also clean .gitignore files and untracked files
-        git fetch --prune --all
+        git fetch --prune --all || echo "git fetch failed" 1>&2 ; exit 1
         git checkout --force $GIT_BRANCH  # force checkout to throwaway local changes
     fi
 

From 40081c7c7f7ba575b57ad7253bc95c96af0d2ed5 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 21:47:45 -0400
Subject: [PATCH 24/26] deploy raven testdata cronjob: change checkout cache to
 under /data for persistence

/tmp will disappear on reboot.
---
 .../components/scheduler/deploy_raven_testdata_to_thredds.env   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
index 574f7a46e..1f5289e97 100644
--- a/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
+++ b/birdhouse/components/scheduler/deploy_raven_testdata_to_thredds.env
@@ -11,7 +11,7 @@ fi
 # Location for local cache of git clone to save bandwidth and time from always
 # re-cloning from scratch.
 if [ -z "$DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE" ]; then
-    DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE="/tmp/deploy_raven_testdata_to_thredds_checkout_cache"
+    DEPLOY_RAVEN_TESTDATA_CHECKOUT_CACHE="/data/deploy_data_cache/deploy_raven_testdata_to_thredds"
 fi
 
 # Location of deploy-data config file.

From 6fab60a5cfcad291bfa40790a0e73938b68df7d4 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 22:07:26 -0400
Subject: [PATCH 25/26] deploy-data: fix error checking always exit even if no
 error

+ git clone https://github.com/Ouranosinc/raven /data/deploy_data_cache/deploy_raven_testdata_to_thredds/raven
Cloning into '/data/deploy_data_cache/deploy_raven_testdata_to_thredds/raven'...
Updating files:  89% (344/383)^MUpdating files:  90% (345/383)^MUpdating files:  91% (349/383)^MUpdating files:  92% (353/383)^MUpdating files:  93% (357/383)^MUpdating files:  94% (361/383)^MUpdating files:  95% (364/383)^MUpdating files:  96% (368/383)^MUpdating files:  97% (372/383)^MUpdating files:  98% (376/383)^MUpdating files:  99% (380/383)^MUpdating files: 100% (383/383)^MUpdating files: 100% (383/383), done.
+ exit 1
---
 birdhouse/deployment/deploy-data | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index 4da17c8fc..aae9c8ba1 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -110,7 +110,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
     CLONE_DEST="$CHECKOUT_CACHE/$GIT_CHECKOUT_NAME"
     if [ ! -d "$CLONE_DEST" ]; then
         echo "checkout repo '$GIT_REPO_URL' on branch '$GIT_BRANCH' to '$CLONE_DEST'"
-        git clone $GIT_REPO_URL $CLONE_DEST || echo "git clone failed" 1>&2 ; exit 1
+        git clone $GIT_REPO_URL $CLONE_DEST || echo "git clone failed" 1>&2 && exit 1
         cd $CLONE_DEST
         git checkout $GIT_BRANCH
     else
@@ -118,7 +118,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
         cd $CLONE_DEST
         git remote -v  # log remote, should match GIT_REPO_URL
         git clean -fdx  # force, recur dir, also clean .gitignore files and untracked files
-        git fetch --prune --all || echo "git fetch failed" 1>&2 ; exit 1
+        git fetch --prune --all || echo "git fetch failed" 1>&2 && exit 1
         git checkout --force $GIT_BRANCH  # force checkout to throwaway local changes
     fi
 

From 60b817f2381d1b734d6b6e86de9fef6d4c114d00 Mon Sep 17 00:00:00 2001
From: Long Vu <vu.long@ouranos.ca>
Date: Wed, 14 Oct 2020 22:26:22 -0400
Subject: [PATCH 26/26] deploy-data: really fix error checking always exit even
 if no error

+ git clone https://github.com/Ouranosinc/raven /data/deploy_data_cache/deploy_raven_testdata_to_thredds/raven
Cloning into '/data/deploy_data_cache/deploy_raven_testdata_to_thredds/raven'...
Updating files: 100% (383/383), done.
+ exit 1
---
 birdhouse/deployment/deploy-data | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/birdhouse/deployment/deploy-data b/birdhouse/deployment/deploy-data
index aae9c8ba1..4dbbba8fe 100755
--- a/birdhouse/deployment/deploy-data
+++ b/birdhouse/deployment/deploy-data
@@ -110,7 +110,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
     CLONE_DEST="$CHECKOUT_CACHE/$GIT_CHECKOUT_NAME"
     if [ ! -d "$CLONE_DEST" ]; then
         echo "checkout repo '$GIT_REPO_URL' on branch '$GIT_BRANCH' to '$CLONE_DEST'"
-        git clone $GIT_REPO_URL $CLONE_DEST || echo "git clone failed" 1>&2 && exit 1
+        git clone $GIT_REPO_URL $CLONE_DEST || exit 1
         cd $CLONE_DEST
         git checkout $GIT_BRANCH
     else
@@ -118,7 +118,7 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
         cd $CLONE_DEST
         git remote -v  # log remote, should match GIT_REPO_URL
         git clean -fdx  # force, recur dir, also clean .gitignore files and untracked files
-        git fetch --prune --all || echo "git fetch failed" 1>&2 && exit 1
+        git fetch --prune --all || exit 1
         git checkout --force $GIT_BRANCH  # force checkout to throwaway local changes
     fi