Publishing and sharing scientific codes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jovyan/agave\n",
      "Requirement already up-to-date: setvar in /opt/conda/lib/python3.6/site-packages\n",
      "\u001b[1;0mToken for sandbox:dooley successfully refreshed and cached for 14400 seconds\n",
      "b16c31608d96303d5955c23ac25050\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!mkdir -p ~/agave\n",
    "\n",
    "%cd ~/agave\n",
    "\n",
    "!pip3 install --upgrade setvar\n",
    "\n",
    "import re\n",
    "import os\n",
    "import sys\n",
    "from setvar import *\n",
    "from time import sleep\n",
    "\n",
    "# This cell enables inline plotting in the notebook\n",
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "!auth-tokens-refresh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding dynamic inputs  \n",
    "\n",
    "Let's update our previous input file with some dynamic behavior. Here we replace a couple of the variables, `AMP` and `WID` with placeholders. We'll also rename it with a `.tpl` extension so we're not confused out its purpose. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing file `funwave-input.tpl'\n"
     ]
    }
   ],
   "source": [
    "writefile(\"funwave-input.tpl\",\"\"\"\n",
    "!INPUT FILE FOR FUNWAVE_TVD\n",
    "  ! NOTE: all input parameter are capital sensitive\n",
    "  ! --------------------TITLE-------------------------------------\n",
    "  ! title only for log file\n",
    "TITLE = VESSEL\n",
    "  ! -------------------HOT START---------------------------------\n",
    "HOT_START = F\n",
    "FileNumber_HOTSTART = 1\n",
    "  ! -------------------PARALLEL INFO-----------------------------\n",
    "  ! \n",
    "  !    PX,PY - processor numbers in X and Y\n",
    "  !    NOTE: make sure consistency with mpirun -np n (px*py)\n",
    "  !    \n",
    "PX = 4\n",
    "PY = 1\n",
    "  ! --------------------DEPTH-------------------------------------\n",
    "  ! Depth types, DEPTH_TYPE=DATA: from depth file\n",
    "  !              DEPTH_TYPE=FLAT: idealized flat, need depth_flat\n",
    "  !              DEPTH_TYPE=SLOPE: idealized slope, \n",
    "  !                                 need slope,SLP starting point, Xslp\n",
    "  !                                 and depth_flat\n",
    "DEPTH_TYPE = FLAT\n",
    "DEPTH_FLAT = 10.0\n",
    "  ! -------------------PRINT---------------------------------\n",
    "  ! PRINT*,\n",
    "  ! result folder\n",
    "RESULT_FOLDER = output/\n",
    "\n",
    "  ! ------------------DIMENSION-----------------------------\n",
    "  ! global grid dimension\n",
    "Mglob = 500\n",
    "Nglob = 100\n",
    "\n",
    "  ! ----------------- TIME----------------------------------\n",
    "  ! time: total computational time/ plot time / screen interval \n",
    "  ! all in seconds\n",
    "TOTAL_TIME = 10.0\n",
    "PLOT_INTV = 1.0\n",
    "PLOT_INTV_STATION = 50000.0\n",
    "SCREEN_INTV = 1.0\n",
    "HOTSTART_INTV = 360000000000.0\n",
    "\n",
    "WAVEMAKER = INI_GAU\n",
    "AMP = %AMP%\n",
    "Xc = 250.0\n",
    "Yc = 50.0\n",
    "WID = %WID%\n",
    "\n",
    "  ! -----------------GRID----------------------------------\n",
    "  ! if use spherical grid, in decimal degrees\n",
    "  ! cartesian grid sizes\n",
    "DX = 1.0\n",
    "DY = 1.0\n",
    "  ! ----------------SHIP WAKES ----------------------------\n",
    "VESSEL_FOLDER = ./\n",
    "NumVessel = 2\n",
    "  ! -----------------OUTPUT-----------------------------\n",
    "ETA = T\n",
    "U = T\n",
    "V = T\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Smarter wrapper templates  \n",
    "\n",
    "Our previous wrapper script was as simple as it gets, blindly calling any command it was given. Here we will add some logic restrict it to only run our funwave application with required variables, print out our runtime variables, and properly exit if anything goes wrong.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing file `funwave-wrapper.txt'\n"
     ]
    }
   ],
   "source": [
    "writefile(\"funwave-wrapper.txt\",\"\"\"\n",
    "#!/bin/bash\n",
    "# This is a generic wrapper script for checking the runtime environment\n",
    "# of a job and verifying the runtime template variable values\n",
    "set +x\n",
    "\n",
    "date\n",
    "\n",
    "# print out the agave runtiem variables\n",
    "echo \"##################################################\"\n",
    "echo \"# Agave Job Runtime Variables \"\n",
    "echo \"##################################################\"\n",
    "echo \"\\n\"\n",
    "echo 'AGAVE_JOB_NAME=\"\\${AGAVE_JOB_NAME}\"'\n",
    "echo 'AGAVE_JOB_ID=\"\\${AGAVE_JOB_ID}\"'\n",
    "echo 'AGAVE_JOB_APP_ID=\"\\${AGAVE_JOB_APP_ID}\"'\n",
    "echo 'AGAVE_JOB_EXECUTION_SYSTEM=\"\\${AGAVE_JOB_EXECUTION_SYSTEM}\"'\n",
    "echo 'AGAVE_JOB_BATCH_QUEUE=\"\\${AGAVE_JOB_BATCH_QUEUE}\"'\n",
    "echo 'AGAVE_JOB_SUBMIT_TIME=\"\\${AGAVE_JOB_SUBMIT_TIME}\"'\n",
    "echo 'AGAVE_JOB_ARCHIVE_SYSTEM=\"\\${AGAVE_JOB_ARCHIVE_SYSTEM}\"'\n",
    "echo 'AGAVE_JOB_ARCHIVE_PATH=\"\\${AGAVE_JOB_ARCHIVE_PATH}\"'\n",
    "echo 'AGAVE_JOB_NODE_COUNT=\"\\${AGAVE_JOB_NODE_COUNT}\"'\n",
    "echo 'AGAVE_JOB_PROCESSORS_PER_NODE=\"\\${AGAVE_JOB_PROCESSORS_PER_NODE}\"'\n",
    "echo 'AGAVE_JOB_MEMORY_PER_NODE=\"\\${AGAVE_JOB_MEMORY_PER_NODE}\"'\n",
    "echo 'AGAVE_JOB_ARCHIVE_URL=\"\\${AGAVE_JOB_ARCHIVE_URL}\"'\n",
    "echo 'AGAVE_JOB_OWNER=\"\\${AGAVE_JOB_OWNER}\"'\n",
    "echo 'AGAVE_JOB_TENANT=\"\\${AGAVE_JOB_TENANT}\"'\n",
    "echo 'AGAVE_JOB_ARCHIVE=\"\\${AGAVE_JOB_ARCHIVE}\"'\n",
    "echo 'AGAVE_JOB_MAX_RUNTIME=\"\\${AGAVE_JOB_MAX_RUNTIME}\"'\n",
    "echo 'AGAVE_JOB_MAX_RUNTIME_SECONDS=\"\\${AGAVE_JOB_MAX_RUNTIME_SECONDS}\"'\n",
    "echo 'AGAVE_JOB_MAX_RUNTIME_MILLISECONDS=\"\\${AGAVE_JOB_MAX_RUNTIME_MILLISECONDS}\"'\n",
    "echo 'AGAVE_BASE_URL=\"\\${AGAVE_BASE_URL}\"'\n",
    "echo 'AGAVE_JOB_ARCHIVE=\"\\${AGAVE_JOB_ARCHIVE}\"'\n",
    "echo 'AGAVE_CACHE_DIR=\"\\${AGAVE_CACHE_DIR}\"'\n",
    "echo 'AGAVE_JOB_ACCESS_TOKEN=\"\\${AGAVE_JOB_ACCESS_TOKEN}\"'\n",
    "echo 'AGAVE_JOB_REFRESH_TOKEN=\"\\${AGAVE_JOB_REFRESH_TOKEN}\"'\n",
    "echo 'AGAVE_JOB_PACKAGE_OUTPUT=\"\\${AGAVE_JOB_PACKAGE_OUTPUT}\"'\n",
    "echo 'AGAVE_JOB_COMPRESS_OUTPUT=\"\\${AGAVE_JOB_COMPRESS_OUTPUT}\"'\n",
    "\n",
    "echo \"##################################################\"\n",
    "echo \"# Job Runtime Environment \"\n",
    "echo \"##################################################\"\n",
    "echo \"\\n\"\n",
    "\n",
    "LD_LIBRARY_PATH=/usr/local/lib \n",
    "\n",
    "# print environment\n",
    "env\n",
    "\n",
    "# copy to file for usage later on\n",
    "env > ./environment.out\n",
    "\n",
    "# resolve our job request parameters in funwave's input file\n",
    "cp -f funwave-input.tpl  input.txt\n",
    "\n",
    "sed -i 's#%WID%#\\${WID}#' input.txt\n",
    "sed -i 's#%AMP%#\\${AMP}#' input.txt\n",
    "\n",
    "# run our funwave command\n",
    "mpirun -np 4 /home/jovyan/FUNWAVE-TVD/src/funwave_vessel \n",
    "\n",
    "# if the job request set the parameter compress_ouput to true\n",
    "# the folder will be compressed\n",
    "if [[ -z \"\\${compress_output}\" ]]; then \n",
    "    tar czf output.tgz output\n",
    "fi\n",
    "\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Agave commands, we make a directory on the storage server an deploy our wrapper file there."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;0mSuccessfully created folder funwave-agave-deployment\u001b[0m\n",
      "Uploading funwave-wrapper.txt...\n",
      "######################################################################## 100.0%\n",
      "Uploading funwave-input.tpl...\n",
      "######################################################################## 100.0%\n"
     ]
    }
   ],
   "source": [
    "!files-mkdir -S ${STORAGE_MACHINE} -N funwave-${DEPLOYMENT_PATH}\n",
    "!files-upload -F funwave-wrapper.txt -S ${STORAGE_MACHINE} funwave-${DEPLOYMENT_PATH}/\n",
    "!files-upload -F funwave-input.tpl -S ${STORAGE_MACHINE} funwave-${DEPLOYMENT_PATH}/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All agave applications require a test file. The test file is a free form text file which allows you to specify what resources you might need to test your application."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing file `funwave-test.txt'\n"
     ]
    }
   ],
   "source": [
    "writefile(\"funwave-test.txt\",\"\"\"\n",
    "compress_output=1\n",
    "AMP=2.0\n",
    "WID=24.0\n",
    "funwave-wrapper.txt\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;0mSuccessfully created folder funwave-agave-deployment\u001b[0m\n",
      "Uploading funwave-test.txt...\n",
      "######################################################################## 100.0%\n"
     ]
    }
   ],
   "source": [
    "!files-mkdir -S ${STORAGE_MACHINE} -N funwave-${DEPLOYMENT_PATH}\n",
    "!files-upload -F funwave-test.txt -S ${STORAGE_MACHINE} funwave-${DEPLOYMENT_PATH}/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## More descriptive apps  \n",
    "\n",
    "We are recycling our previous app description here with a few changes. First, we are updating the app id so a new app will be created. Second we have added three new parameters.  \n",
    "\n",
    "* `compress_output` is a boolean parameter that tells the wrapper whether to compress the output folder after running.   \n",
    "* `AMP` is a numeric parameter specifying the wave amplitude. Notice the validator requires this to be a decimal value. \n",
    "* `WID` is a numeric parameter specifying the width. Notice the validator requires this to be a decimal value.  \n",
    "\n",
    "We have also removed the data file input from the previous app description. This is because we uploaded the input template with our app description, so it will be copied in and dynamically crated based on the runtime variable values every time the app is run. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing file `funwave-app.txt'\n"
     ]
    }
   ],
   "source": [
    "writefile(\"funwave-app.txt\",\"\"\"\n",
    "{  \n",
    "   \"name\":\"${AGAVE_USERNAME}-${MACHINE_NAME}-funwave\",\n",
    "   \"version\":\"1.0\",\n",
    "   \"label\":\"Runs a command\",\n",
    "   \"shortDescription\":\"Runs a command\",\n",
    "   \"longDescription\":\"\",\n",
    "   \"deploymentSystem\":\"${STORAGE_MACHINE}\",\n",
    "   \"deploymentPath\":\"funwave-${DEPLOYMENT_PATH}\",\n",
    "   \"templatePath\":\"funwave-wrapper.txt\",\n",
    "   \"testPath\":\"funwave-test.txt\",\n",
    "   \"executionSystem\":\"${EXEC_MACHINE}\",\n",
    "   \"executionType\":\"CLI\",\n",
    "   \"parallelism\":\"SERIAL\",\n",
    "   \"modules\":[],\n",
    "   \"inputs\":[],\n",
    "   \"parameters\":[{\n",
    "     \"id\" : \"compress_output\",\n",
    "     \"value\" : {\n",
    "       \"visible\":true,\n",
    "       \"required\":true,\n",
    "       \"type\":\"bool\",\n",
    "       \"order\":0,\n",
    "       \"enquote\":false,\n",
    "       \"default\":true\n",
    "     },\n",
    "     \"details\":{\n",
    "         \"label\": \"Compress output folder\",\n",
    "         \"description\": \"If true, output will be packed and compressed\",\n",
    "         \"argument\": null,\n",
    "         \"showArgument\": false,\n",
    "         \"repeatArgument\": false\n",
    "     },\n",
    "     \"semantics\":{\n",
    "         \"argument\": null,\n",
    "         \"showArgument\": false,\n",
    "         \"repeatArgument\": false\n",
    "     }\n",
    "   },\n",
    "   {\n",
    "     \"id\" : \"AMP\",\n",
    "     \"value\" : {\n",
    "       \"visible\":true,\n",
    "       \"required\":true,\n",
    "       \"type\":\"string\",\n",
    "       \"order\":0,\n",
    "       \"enquote\":false,\n",
    "       \"default\":\"3.0\",\n",
    "       \"validator\": \"\\\\\\\\d+\\\\\\\\.\\\\\\\\d+\"\n",
    "     },\n",
    "     \"details\":{\n",
    "         \"label\": \"Wave amplitude\",\n",
    "         \"description\": \"Wave amplitude expressed as a decimal value\",\n",
    "         \"argument\": null,\n",
    "         \"showArgument\": false,\n",
    "         \"repeatArgument\": false\n",
    "     },\n",
    "     \"semantics\":{\n",
    "         \"argument\": null,\n",
    "         \"showArgument\": false,\n",
    "         \"repeatArgument\": false\n",
    "     }\n",
    "   },\n",
    "   {\n",
    "     \"id\" : \"WID\",\n",
    "     \"value\" : {\n",
    "       \"visible\":true,\n",
    "       \"required\":true,\n",
    "       \"type\":\"string\",\n",
    "       \"order\":0,\n",
    "       \"enquote\":false,\n",
    "       \"default\":\"25.0\",\n",
    "       \"validator\": \"\\\\\\\\d+\\\\\\\\.\\\\\\\\d+\"\n",
    "     },\n",
    "     \"details\":{\n",
    "         \"label\": \"Width\",\n",
    "         \"description\": \"Width expressed as a decimal value\",\n",
    "         \"argument\": null,\n",
    "         \"showArgument\": false,\n",
    "         \"repeatArgument\": false\n",
    "     },\n",
    "     \"semantics\":{\n",
    "         \"argument\": null,\n",
    "         \"showArgument\": false,\n",
    "         \"repeatArgument\": false\n",
    "     }\n",
    "   }],\n",
    "   \"outputs\":[]\n",
    "}\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;0m\u001b[1;0mSuccessfully added app dooley-nectar-funwave-1.0\u001b[0m\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!apps-addupdate -F funwave-app.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running Jobs  \n",
    "\n",
    "We have registered a new Agave app to run our funwave code. Let's update our job definition to test it out. For fun, let's post our notifications to Slack. If you do not have access to a Slack channel where you can create an incoming webhook, use the previous requestbin url from the job request file below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WEBHOOK_URL=https://hooks.slack.com/services/\n",
      "WEBHOOK_URL=https://requestbin.agaveapi.co/13s48n91?name=${JOB_NAME}&event=${EVENT}&jobid=${JOB_ID}\n"
     ]
    }
   ],
   "source": [
    "setvar(\"\"\"\n",
    "WEBHOOK_URL=https://hooks.slack.com/services/\n",
    "WEBHOOK_URL=${REQUESTBIN_URL}?name=\\${JOB_NAME}&event=\\${EVENT}&jobid=\\${JOB_ID}\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing file `funwave-job.txt'\n"
     ]
    }
   ],
   "source": [
    "writefile(\"funwave-job.txt\",\"\"\"\n",
    " {\n",
    "   \"name\":\"funwave-1\",\n",
    "   \"appId\": \"${AGAVE_USERNAME}-${MACHINE_NAME}-funwave-1.0\",\n",
    "   \"executionSystem\": \"${EXEC_MACHINE}\",\n",
    "   \"maxRunTime\":\"00:10:00\",\n",
    "   \"archive\": false,\n",
    "   \"notifications\": [\n",
    "    {\n",
    "      \"url\":\"${WEBHOOK_URL}\",\n",
    "      \"event\":\"*\",\n",
    "      \"persistent\":\"true\"\n",
    "    }\n",
    "   ],\n",
    "   \"parameters\": {\n",
    "     \"compress_output\":true,\n",
    "     \"AMP\":\"4.0\",\n",
    "     \"WID\":\"20.0\"\n",
    "   }\n",
    " }\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because the setvar() command can evalute `$()` style bash shell substitutions, we will use it to submit our job. This will capture the output of the submit command, and allow us to parse it for the JOB_ID. We'll use the JOB_ID in several subsequent steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OUTPUT=Successfully submitted job 954095645475008025-242ac114-0001-007\n",
      "JOB_ID=954095645475008025-242ac114-0001-007\n"
     ]
    }
   ],
   "source": [
    "setvar(\"\"\"\n",
    "# Capture the output of the job submit command\n",
    "OUTPUT=$(jobs-submit -F funwave-job.txt)\n",
    "# Parse out the job id from the output\n",
    "JOB_ID=$(echo $OUTPUT | cut -d' ' -f4)\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Job Monitoring and Output</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "While the job is running, the requestbin you registered will receive webhooks from Agave every time a job event occurs. To monitor this in real time, evaluate the next cell an visit the printed url in your browser:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://requestbin.agaveapi.co/13s48n91?inspect\r\n"
     ]
    }
   ],
   "source": [
    "!echo ${REQUESTBIN_URL}?inspect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Of course, you can also monitor the job status by polling. Note that the notifications you receive via email and webhook are less wasteful of resources. However, we show you this for completeness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=SUBMITTING\n",
      "STAT=FAILED\n"
     ]
    }
   ],
   "source": [
    "for iter in range(20):\n",
    "    setvar(\"STAT=$(jobs-status $JOB_ID)\")\n",
    "    stat = os.environ[\"STAT\"]\n",
    "    sleep(5.0)\n",
    "    if stat == \"FINISHED\" or stat == \"FAILED\":\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The jobs-history command provides you a record of the steps of what your job did. If your job fails for some reason, this is your best diagnostic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jobs-history 3098053922098909671-242ac114-0001-007\n",
      "\u001b[1;0mJob accepted and queued for submission.\n",
      "Skipping staging. No input data associated with this job.\n",
      "Preparing job for submission.\n",
      "Attempt 1 to submit job\n",
      "Fetching app assets from agave://nectar-storage-dooley/funwave-agave-deployment\n",
      "Staging runtime assets to agave://nectar-exec-dooley//home/jovyan/dooley/job-3098053922098909671-242ac114-0001-007-funwave-1\n",
      "Job started running\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!echo jobs-history ${JOB_ID}\n",
    "!jobs-history ${JOB_ID}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This command shows you the job id's and status of the last 5 jobs you ran."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;0m4930048995024441831-242ac114-0001-007 FINISHED\r\n",
      "8056752744420011545-242ac114-0001-007 FINISHED\r\n",
      "8508968551844286951-242ac114-0001-007 FINISHED\r\n",
      "7560051115402718745-242ac114-0001-007 FINISHED\r\n",
      "3339748224467472871-242ac114-0001-007 FINISHED\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!jobs-list -l 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This next command provides you with a list of all the files generated by your job. You can use it to figure out which files you want to retrieve with jobs-output-get."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;0m| type | length | name                |\r\n",
      "| ---- | ------ | ----                |\r\n",
      "| file | 101    | .agave.archive      |\r\n",
      "| file | 232    | .agave.log          |\r\n",
      "| file | 607    | environment.out     |\r\n",
      "| file | 29     | fork-test.txt       |\r\n",
      "| file | 197    | funwave-1.err       |\r\n",
      "| file | 4654   | funwave-1.ipcexe    |\r\n",
      "| file | 10139  | funwave-1.out       |\r\n",
      "| file | 5      | funwave-1.pid       |\r\n",
      "| file | 1757   | funwave-input.tpl   |\r\n",
      "| file | 54     | funwave-test.txt    |\r\n",
      "| file | 2455   | funwave-wrapper.txt |\r\n",
      "| file | 178    | Grid_Range.out      |\r\n",
      "| file | 1754   | input.txt           |\r\n",
      "| file | 8775   | LOG.txt             |\r\n",
      "| dir  | 4096   | output              |\r\n",
      "| file | 583    | time_dt.out         |\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!jobs-output-list --rich --filter=type,length,name ${JOB_ID}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrieve the standard output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "######################################################################## 100.0%\n",
      "Sat Nov  4 23:05:00 UTC 2017\n",
      "##################################################\n",
      "# Agave Job Runtime Variables \n",
      "##################################################\n",
      "\n",
      "\n",
      "AGAVE_JOB_NAME=\"funwave-1\"\n",
      "AGAVE_JOB_ID=\"4930048995024441831-242ac114-0001-007\"\n",
      "AGAVE_JOB_APP_ID=\"dooley-nectar-funwave-1.0\"\n",
      "AGAVE_JOB_EXECUTION_SYSTEM=\"nectar-exec-dooley\"\n",
      "AGAVE_JOB_BATCH_QUEUE=\"none\"\n",
      "AGAVE_JOB_SUBMIT_TIME=\"2017-11-04T18:03:02.000-05:00\"\n",
      "AGAVE_JOB_ARCHIVE_SYSTEM=\"\"\n",
      "AGAVE_JOB_ARCHIVE_PATH=\"dooley/archive/jobs/job-4930048995024441831-242ac114-0001-007\"\n",
      "AGAVE_JOB_NODE_COUNT=\"1\"\n",
      "AGAVE_JOB_PROCESSORS_PER_NODE=\"1\"\n",
      "AGAVE_JOB_MEMORY_PER_NODE=\"1.0\"\n",
      "AGAVE_JOB_ARCHIVE_URL=\"\"\n",
      "AGAVE_JOB_OWNER=\"dooley\"\n",
      "AGAVE_JOB_TENANT=\"sandbox\"\n",
      "AGAVE_JOB_ARCHIVE=\"\"\n",
      "AGAVE_JOB_MAX_RUNTIME=\"00:30:00\"\n",
      "AGAVE_JOB_MAX_RUNTIME_SECONDS=\"1800.0\"\n",
      "AGAVE_JOB_MAX_RUNTIME_MILLISECONDS=\"1800000\"\n",
      "AGAVE_BASE_URL=\"https://agave-auth.solveij.com/\"\n",
      "AGAVE_JOB_ARCHIVE=\"\"\n",
      "AGAVE_CACHE_DIR=\"${AGAVE_CACHE_DIR}\"\n",
      "AGAVE_JOB_ACCESS_TOKEN=\"${AGAVE_JOB_ACCESS_TOKEN}\"\n",
      "AGAVE_JOB_REFRESH_TOKEN=\"${AGAVE_JOB_REFRESH_TOKEN}\"\n",
      "AGAVE_JOB_PACKAGE_OUTPUT=\"${AGAVE_JOB_PACKAGE_OUTPUT}\"\n",
      "AGAVE_JOB_COMPRESS_OUTPUT=\"${AGAVE_JOB_COMPRESS_OUTPUT}\"\n",
      "##################################################\n",
      "# Job Runtime Environment \n",
      "##################################################\n",
      "\n",
      "\n",
      "SHELL=/bin/bash\n",
      "TERM=dumb\n",
      "SSH_CLIENT=127.0.0.1 38718 22\n",
      "OLDPWD=/home/jovyan/dooley/job-4930048995024441831-242ac114-0001-007-funwave-1\n",
      "SSH_TTY=/dev/pts/1\n",
      "USER=jovyan\n",
      "LS_COLORS=\n",
      "LD_LIBRARY_PATH=/usr/local/lib\n",
      "PATH=/home/jovyan/bin:/home/jovyan/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games\n",
      "MAIL=/var/mail/jovyan\n",
      "PWD=/home/jovyan/dooley/job-4930048995024441831-242ac114-0001-007-funwave-1\n",
      "HOME=/home/jovyan\n",
      "SHLVL=2\n",
      "LOGNAME=jovyan\n",
      "SSH_CONNECTION=127.0.0.1 38718 127.0.0.1 22\n",
      "LESSOPEN=| /usr/bin/lesspipe %s\n",
      "LESSCLOSE=/usr/bin/lesspipe %s %s\n",
      "_=/usr/bin/env\n",
      " BATHY_CORRECTION DOES NOT EXIST. USE DEFAULT VALUE\n",
      " INI_UVZ DOES NOT EXIST. USE DEFAULT VALUE\n",
      " WaterLevel DOES NOT EXIST. USE DEFAULT VALUE\n",
      " ETA_LIMITER DOES NOT EXIST. USE DEFAULT VALUE\n",
      " EqualEnergy DOES NOT EXIST. USE DEFAULT VALUE\n",
      " PERIODIC DOES NOT EXIST. USE DEFAULT VALUE\n",
      " DIFFUSION_SPONGE DOES NOT EXIST. USE DEFAULT VALUE\n",
      " DIRECT_SPONGE DOES NOT EXIST. USE DEFAULT VALUE\n",
      " FRICTION_SPONGE DOES NOT EXIST. USE DEFAULT VALUE\n",
      " OBSTACLE_FILE DOES NOT EXIST. USE DEFAULT VALUE\n",
      " BREAKWATER_FILE DOES NOT EXIST. USE DEFAULT VALUE\n",
      " DISPERSION DOES NOT EXIST. USE DEFAULT VALUE\n",
      "         DISPERSION Default:  DISPERSION\n",
      " Gamma1 DOES NOT EXIST. USE DEFAULT VALUE\n",
      "        Gamma1 Default:  1.0: DISPERSION\n",
      " Gamma2 DOES NOT EXIST. USE DEFAULT VALUE\n",
      "              Gamma2 Default:  1.0: Full nonlinear\n",
      " Beta_ref DOES NOT EXIST. USE DEFAULT VALUE\n",
      "               Beta_ref Default:  -0.531\n",
      " Gamma3 DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                      Gamma3 Default:  1.0: NOT fully linear\n",
      " VISCOSITY_BREAKING DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                   VISCOSITY_BREAKING Default:  SWE Breaking\n",
      " SWE_ETA_DEP DOES NOT EXIST. USE DEFAULT VALUE\n",
      "               SWE_ETA_DEP Default:  0.8\n",
      " Friction_Matrix DOES NOT EXIST. USE DEFAULT VALUE\n",
      "             Friction_Matrix Default:  constant Cd\n",
      " Cd DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                  Cd_fixed Default:  0.0\n",
      " Time_Scheme DOES NOT EXIST. USE DEFAULT VALUE\n",
      "       Time_Scheme Default:  Runge_Kutta\n",
      " CONSTRUCTION DOES NOT EXIST. USE DEFAULT VALUE\n",
      " HIGH_ORDER DOES NOT EXIST. USE DEFAULT VALUE\n",
      "  HIGH_ORDER                     NOT DEFINED, USE FOURTH-ORDER\n",
      " CFL DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                       CFL Default:  0.5\n",
      " DT_fixed DOES NOT EXIST. USE DEFAULT VALUE\n",
      " FroudeCap DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                 FroudeCap Default:  3.0\n",
      " MinDepth DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                MinDepth Default:  0.1 m\n",
      " MinDepthFrc DOES NOT EXIST. USE DEFAULT VALUE\n",
      "             MinDepthFrc Default:  0.1 m\n",
      " SHOW_BREAKING DOES NOT EXIST. USE DEFAULT VALUE\n",
      "            SHOW_BREAKING Default:  TRUE\n",
      " Cbrk1 DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                    Cbrk1 Default:  0.65\n",
      " Cbrk2 DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                    Cbrk2 Default:  0.35\n",
      " WAVEMAKER_Cbrk DOES NOT EXIST. USE DEFAULT VALUE\n",
      "            WAVEMAKER_Cbrk Default:  1.0\n",
      " WAVEMAKER_VIS DOES NOT EXIST. USE DEFAULT VALUE\n",
      "           WAVEMAKER_VIS Default:  FALSE\n",
      " T_INTV_mean DOES NOT EXIST. USE DEFAULT VALUE\n",
      "             T_INTV_mean Default:  LARGE\n",
      " STEADY_TIME DOES NOT EXIST. USE DEFAULT VALUE\n",
      "             STEADY_TIME Default:  LARGE\n",
      " C_smg DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                     C_smg Default:  0.0\n",
      " nu_bkg DOES NOT EXIST. USE DEFAULT VALUE\n",
      "                    nu_bkg Default:  0.0\n",
      " FIELD_IO_TYPE DOES NOT EXIST. USE DEFAULT VALUE\n",
      " NumberStations DOES NOT EXIST. USE DEFAULT VALUE\n",
      " OUTPUT_RES DOES NOT EXIST. USE DEFAULT VALUE\n",
      "         OUTPUT_RES NOT FOUND, OUTPUT_RES=1: full resolution\n",
      " DEPTH_OUT DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Hmax DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Hmin DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Umax DOES NOT EXIST. USE DEFAULT VALUE\n",
      " MFmax DOES NOT EXIST. USE DEFAULT VALUE\n",
      " VORmax DOES NOT EXIST. USE DEFAULT VALUE\n",
      " MASK DOES NOT EXIST. USE DEFAULT VALUE\n",
      " MASK9 DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Umean DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Vmean DOES NOT EXIST. USE DEFAULT VALUE\n",
      " ETAmean DOES NOT EXIST. USE DEFAULT VALUE\n",
      " WaveHeight DOES NOT EXIST. USE DEFAULT VALUE\n",
      " SXL DOES NOT EXIST. USE DEFAULT VALUE\n",
      " SXR DOES NOT EXIST. USE DEFAULT VALUE\n",
      " SYL DOES NOT EXIST. USE DEFAULT VALUE\n",
      " SYR DOES NOT EXIST. USE DEFAULT VALUE\n",
      " SourceX DOES NOT EXIST. USE DEFAULT VALUE\n",
      " SourceY DOES NOT EXIST. USE DEFAULT VALUE\n",
      " P DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Q DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Fx DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Fy DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Gx DOES NOT EXIST. USE DEFAULT VALUE\n",
      " Gy DOES NOT EXIST. USE DEFAULT VALUE\n",
      " AGE DOES NOT EXIST. USE DEFAULT VALUE\n",
      " OUT_NU DOES NOT EXIST. USE DEFAULT VALUE\n",
      " TMP DOES NOT EXIST. USE DEFAULT VALUE\n",
      "      EtaBlowVal Default:  100xmax_depth\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.4266E-01  0.4266E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2583E+04  0.6410E+07  0.3995E+01 -0.3338E-06  0.5200E-01  0.5199E-01\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.5200E-01  0.1090E+02  0.4771E-02\n",
      "   PRINTING FILE NO.     1  TIME/TOTAL:        0.043 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.1022E+01  0.4249E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2569E+04  0.6406E+07  0.2451E+01 -0.8911E-06  0.9724E+00  0.9720E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.9739E+00  0.1069E+02  0.9108E-01\n",
      "   PRINTING FILE NO.     2  TIME/TOTAL:        1.022 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.2005E+01  0.4317E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2542E+04  0.6402E+07  0.9894E+00  0.1968E-05  0.1206E+01  0.1201E+01\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.1210E+01  0.1035E+02  0.1168E+00\n",
      "   PRINTING FILE NO.     3  TIME/TOTAL:        2.005 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.3012E+01  0.4431E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2520E+04  0.6401E+07  0.9698E+00 -0.2528E-05  0.9685E+00  0.9088E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.9715E+00  0.1029E+02  0.9437E-01\n",
      "   PRINTING FILE NO.     4  TIME/TOTAL:        3.012 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.4041E+01  0.4515E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2516E+04  0.6401E+07  0.1367E+01 -0.2694E-05  0.7978E+00  0.5050E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.7997E+00  0.1027E+02  0.7790E-01\n",
      "   PRINTING FILE NO.     5  TIME/TOTAL:        4.041 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.5035E+01  0.4482E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2517E+04  0.6401E+07  0.1373E+01 -0.3279E-05  0.6988E+00  0.4493E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.7030E+00  0.1025E+02  0.6857E-01\n",
      "   PRINTING FILE NO.     6  TIME/TOTAL:        5.035 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.6015E+01  0.4443E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2511E+04  0.6400E+07  0.1210E+01 -0.3242E-05  0.7852E+00  0.8210E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.8210E+00  0.9907E+01  0.8288E-01\n",
      "   PRINTING FILE NO.     7  TIME/TOTAL:        6.015 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.7037E+01  0.4448E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2499E+04  0.6398E+07  0.1089E+01 -0.3322E-05  0.8160E+00  0.9115E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.9115E+00  0.9912E+01  0.9196E-01\n",
      "   PRINTING FILE NO.     8  TIME/TOTAL:        7.037 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.8018E+01  0.4468E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2476E+04  0.6396E+07  0.9959E+00 -0.2938E-05  0.8050E+00  0.7051E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.8050E+00  0.1037E+02  0.7760E-01\n",
      "   PRINTING FILE NO.     9  TIME/TOTAL:        8.018 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.9004E+01  0.4492E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2418E+04  0.6391E+07  0.9197E+00 -0.2615E-05  0.7783E+00  0.5294E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.7783E+00  0.1034E+02  0.7526E-01\n",
      "   PRINTING FILE NO.    10  TIME/TOTAL:        9.004 /      10.000\n",
      " ----------------- STATISTICS ----------------\n",
      "  TIME        DT\n",
      "  0.1004E+02  0.4519E-01\n",
      "  MassVolume  Energy      MaxEta      MinEta      Max U       Max V \n",
      "  0.2282E+04  0.6378E+07  0.8742E+00 -0.3204E-06  0.7448E+00  0.3901E+00\n",
      "  MaxTotalU   PhaseS      Froude \n",
      "  0.7448E+00  0.1031E+02  0.7223E-01\n",
      "   PRINTING FILE NO.    11  TIME/TOTAL:       10.040 /      10.000\n",
      " Simulation takes   53.115449639968574      seconds\n",
      " Normal Termination!\n"
     ]
    }
   ],
   "source": [
    "!jobs-output-get ${JOB_ID} funwave-1.out\n",
    "!cat funwave-1.out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check our resolved wrapper template to verify the variables were replaced."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "######################################################################## 100.0%\n",
      "#!/bin/bash \n",
      "\n",
      "############################################################## \n",
      "# Agave Runtime IO Redirections \n",
      "############################################################## \n",
      "\n",
      "# Capture the PID of this job on the system for remote monitoring \n",
      "echo $$ > funwave-1.pid \n",
      "\n",
      "# Pause for 2 seconds to avoid a callback race condition \n",
      "sleep 2 \n",
      "\n",
      "# Redirect STDERR and STDOUT to the custom job output and error files \n",
      "exec 2>funwave-1.err 1>funwave-1.out \n",
      "\n",
      "\n",
      "##########################################################\n",
      "# Agave Environment Settings \n",
      "##########################################################\n",
      "\n",
      "# Ensure we're in the job work directory \n",
      "cd /home/jovyan/dooley/job-4930048995024441831-242ac114-0001-007-funwave-1\n",
      "\n",
      "# Location of agave job lifecycle log file \n",
      "AGAVE_LOG_FILE=/home/jovyan/dooley/job-4930048995024441831-242ac114-0001-007-funwave-1/.agave.log\n",
      "\n",
      "\n",
      "##########################################################\n",
      "# Agave Utility functions \n",
      "##########################################################\n",
      "\n",
      "# cross-plaltform function to print an ISO8601 formatted timestamp \n",
      "function agave_datetime_iso() { \n",
      "  date '+%Y-%m-%dT%H:%M:%S%z'; \n",
      "} \n",
      "\n",
      "# standard logging function to write agave job lifecycle logs\n",
      "function agave_log_response() { \n",
      "  echo \"[$(agave_datetime_iso)] ${@}\"; \n",
      "} 2>&1 >> \"${AGAVE_LOG_FILE}\"\n",
      "\n",
      "# Callback to signal the job has started executing user-defined logic\n",
      "agave_log_response $(curl -sSk \"https://agave-auth.solveij.com/jobs/v2/trigger/job/4930048995024441831-242ac114-0001-007/token/fb52b791-b0b0-4df4-82ec-bc8ac01fab2d/status/RUNNING?filter=id,status\" 2>&1) \n",
      "\n",
      "\n",
      "##########################################################\n",
      "# Agave App and System Environment Settings \n",
      "##########################################################\n",
      "\n",
      "# No modules commands configured for this app\n",
      "\n",
      "# No custom environment variables configured for this app\n",
      "\n",
      "\n",
      "##########################################################\n",
      "# Begin App Wrapper Template Logic \n",
      "##########################################################\n",
      "\n",
      "#!/bin/bash\n",
      "# This is a generic wrapper script for checking the runtime environment\n",
      "# of a job and verifying the runtime template variable values\n",
      "set +x\n",
      "\n",
      "date\n",
      "\n",
      "# print out the agave runtiem variables\n",
      "echo \"##################################################\"\n",
      "echo \"# Agave Job Runtime Variables \"\n",
      "echo \"##################################################\"\n",
      "echo \"\n",
      "\"\n",
      "echo 'AGAVE_JOB_NAME=\"funwave-1\"'\n",
      "echo 'AGAVE_JOB_ID=\"4930048995024441831-242ac114-0001-007\"'\n",
      "echo 'AGAVE_JOB_APP_ID=\"dooley-nectar-funwave-1.0\"'\n",
      "echo 'AGAVE_JOB_EXECUTION_SYSTEM=\"nectar-exec-dooley\"'\n",
      "echo 'AGAVE_JOB_BATCH_QUEUE=\"none\"'\n",
      "echo 'AGAVE_JOB_SUBMIT_TIME=\"2017-11-04T18:03:02.000-05:00\"'\n",
      "echo 'AGAVE_JOB_ARCHIVE_SYSTEM=\"\"'\n",
      "echo 'AGAVE_JOB_ARCHIVE_PATH=\"dooley/archive/jobs/job-4930048995024441831-242ac114-0001-007\"'\n",
      "echo 'AGAVE_JOB_NODE_COUNT=\"1\"'\n",
      "echo 'AGAVE_JOB_PROCESSORS_PER_NODE=\"1\"'\n",
      "echo 'AGAVE_JOB_MEMORY_PER_NODE=\"1.0\"'\n",
      "echo 'AGAVE_JOB_ARCHIVE_URL=\"\"'\n",
      "echo 'AGAVE_JOB_OWNER=\"dooley\"'\n",
      "echo 'AGAVE_JOB_TENANT=\"sandbox\"'\n",
      "echo 'AGAVE_JOB_ARCHIVE=\"\"'\n",
      "echo 'AGAVE_JOB_MAX_RUNTIME=\"00:30:00\"'\n",
      "echo 'AGAVE_JOB_MAX_RUNTIME_SECONDS=\"1800.0\"'\n",
      "echo 'AGAVE_JOB_MAX_RUNTIME_MILLISECONDS=\"1800000\"'\n",
      "echo 'AGAVE_BASE_URL=\"https://agave-auth.solveij.com/\"'\n",
      "echo 'AGAVE_JOB_ARCHIVE=\"\"'\n",
      "echo 'AGAVE_CACHE_DIR=\"${AGAVE_CACHE_DIR}\"'\n",
      "echo 'AGAVE_JOB_ACCESS_TOKEN=\"${AGAVE_JOB_ACCESS_TOKEN}\"'\n",
      "echo 'AGAVE_JOB_REFRESH_TOKEN=\"${AGAVE_JOB_REFRESH_TOKEN}\"'\n",
      "echo 'AGAVE_JOB_PACKAGE_OUTPUT=\"${AGAVE_JOB_PACKAGE_OUTPUT}\"'\n",
      "echo 'AGAVE_JOB_COMPRESS_OUTPUT=\"${AGAVE_JOB_COMPRESS_OUTPUT}\"'\n",
      "\n",
      "echo \"##################################################\"\n",
      "echo \"# Job Runtime Environment \"\n",
      "echo \"##################################################\"\n",
      "echo \"\n",
      "\"\n",
      "\n",
      "LD_LIBRARY_PATH=/usr/local/lib \n",
      "\n",
      "# print environment\n",
      "env\n",
      "\n",
      "# copy to file for usage later on\n",
      "env > ./environment.out\n",
      "\n",
      "# resolve our job request parameters in funwave's input file\n",
      "cp -f funwave-input.tpl  input.txt\n",
      "\n",
      "sed -i 's#%WID%#20.0#' input.txt\n",
      "sed -i 's#%AMP%#4.0#' input.txt\n",
      "\n",
      "# run our funwave command\n",
      "mpirun -np 4 /home/jovyan/FUNWAVE-TVD/src/funwave_vessel \n",
      "\n",
      "# if the job request set the parameter compress_ouput to true\n",
      "# the folder will be compressed\n",
      "if [[ -z \"1]]; then \n",
      "    tar czf output.tgz output\n",
      "fi\n",
      "\n",
      "\n",
      "##########################################################\n",
      "# End App Wrapper Template Logic \n",
      "##########################################################\n",
      "\n",
      "# Callback to signal the job has completed all user-defined logic\n",
      "agave_log_response $(curl -sSk \"https://agave-auth.solveij.com/jobs/v2/trigger/job/4930048995024441831-242ac114-0001-007/token/fb52b791-b0b0-4df4-82ec-bc8ac01fab2d/status/CLEANING_UP?filter=id,status\" 2>&1) \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!jobs-output-get ${JOB_ID} funwave-1.ipcexe\n",
    "!cat funwave-1.ipcexe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the input.txt file resolved from the input template with our runtime values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "######################################################################## 100.0%\n",
      "!INPUT FILE FOR FUNWAVE_TVD\n",
      "  ! NOTE: all input parameter are capital sensitive\n",
      "  ! --------------------TITLE-------------------------------------\n",
      "  ! title only for log file\n",
      "TITLE = VESSEL\n",
      "  ! -------------------HOT START---------------------------------\n",
      "HOT_START = F\n",
      "FileNumber_HOTSTART = 1\n",
      "  ! -------------------PARALLEL INFO-----------------------------\n",
      "  ! \n",
      "  !    PX,PY - processor numbers in X and Y\n",
      "  !    NOTE: make sure consistency with mpirun -np n (px*py)\n",
      "  !    \n",
      "PX = 4\n",
      "PY = 1\n",
      "  ! --------------------DEPTH-------------------------------------\n",
      "  ! Depth types, DEPTH_TYPE=DATA: from depth file\n",
      "  !              DEPTH_TYPE=FLAT: idealized flat, need depth_flat\n",
      "  !              DEPTH_TYPE=SLOPE: idealized slope, \n",
      "  !                                 need slope,SLP starting point, Xslp\n",
      "  !                                 and depth_flat\n",
      "DEPTH_TYPE = FLAT\n",
      "DEPTH_FLAT = 10.0\n",
      "  ! -------------------PRINT---------------------------------\n",
      "  ! PRINT*,\n",
      "  ! result folder\n",
      "RESULT_FOLDER = output/\n",
      "\n",
      "  ! ------------------DIMENSION-----------------------------\n",
      "  ! global grid dimension\n",
      "Mglob = 500\n",
      "Nglob = 100\n",
      "\n",
      "  ! ----------------- TIME----------------------------------\n",
      "  ! time: total computational time/ plot time / screen interval \n",
      "  ! all in seconds\n",
      "TOTAL_TIME = 10.0\n",
      "PLOT_INTV = 1.0\n",
      "PLOT_INTV_STATION = 50000.0\n",
      "SCREEN_INTV = 1.0\n",
      "HOTSTART_INTV = 360000000000.0\n",
      "\n",
      "WAVEMAKER = INI_GAU\n",
      "AMP = 4.0\n",
      "Xc = 250.0\n",
      "Yc = 50.0\n",
      "WID = 20.0\n",
      "\n",
      "  ! -----------------GRID----------------------------------\n",
      "  ! if use spherical grid, in decimal degrees\n",
      "  ! cartesian grid sizes\n",
      "DX = 1.0\n",
      "DY = 1.0\n",
      "  ! ----------------SHIP WAKES ----------------------------\n",
      "VESSEL_FOLDER = ./\n",
      "NumVessel = 2\n",
      "  ! -----------------OUTPUT-----------------------------\n",
      "ETA = T\n",
      "U = T\n",
      "V = T"
     ]
    }
   ],
   "source": [
    "!jobs-output-get ${JOB_ID} input.txt\n",
    "!cat input.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrieve the standard error output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r\n"
     ]
    }
   ],
   "source": [
    "!jobs-output-get ${JOB_ID} funwave-1.err\n",
    "!cat funwave-1.err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}