Merge pull request #1341 from isb-cgc/isb-cgc-prod-sp

Sprint 35

.circleci/config.yml

@@ -96,6 +96,7 @@ jobs:
           command: |
             sudo -E rm -rf ./lib
             sudo -E /bin/bash ./shell/gcloud_authenticate.sh
+            sudo -E ./google-cloud-sdk/bin/gcloud components update --version 251.0.0
             sudo -E /bin/bash ./shell/unpack_for_deployment.sh
             sudo -E ./google-cloud-sdk/bin/gcloud app deploy --verbosity=debug ./app.yaml --quiet
 workflows:

Dockerfile

@@ -20,6 +20,16 @@
 # single application.
 FROM gcr.io/google_appengine/python
 
+# Create a virtualenv for dependencies. This isolates these packages from
+# system-level packages.
+# Use -p python3 or -p python3.7 to select python version. Default is version 2.
+RUN virtualenv /env -p python3
+
+# Setting these environment variables are the same as running
+# source /env/bin/activate.
+ENV VIRTUAL_ENV /env
+ENV PATH /env/bin:$PATH
+
 RUN apt-get update
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get install -y wget

GenespotRE/settings.py

@@ -78,8 +78,6 @@
 
 # Data Buckets
 OPEN_DATA_BUCKET        = os.environ.get('OPEN_DATA_BUCKET', '')
-DCC_CONTROLLED_DATA_BUCKET = os.environ.get('DCC_CONTROLLED_DATA_BUCKET', '')
-CGHUB_CONTROLLED_DATA_BUCKET = os.environ.get('CGHUB_CONTROLLED_DATA_BUCKET', '')
 GCLOUD_BUCKET           = os.environ.get('GOOGLE_STORAGE_BUCKET')
 
 # BigQuery cohort storage settings
@@ -453,8 +451,6 @@ def GET_BQ_COHORT_SETTINGS():
 OPEN_ACL_GOOGLE_GROUP                   = os.environ.get('OPEN_ACL_GOOGLE_GROUP', '')
 GOOGLE_GROUP_ADMIN                      = os.environ.get('GOOGLE_GROUP_ADMIN', '')
 SUPERADMIN_FOR_REPORTS                  = os.environ.get('SUPERADMIN_FOR_REPORTS', '')
-ERA_LOGIN_URL                           = os.environ.get('ERA_LOGIN_URL', '')
-SAML_FOLDER                             = os.environ.get('SAML_FOLDER', '')
 
 # TaskQueue used when users go through the ERA flow
 LOGOUT_WORKER_TASKQUEUE                  = os.environ.get('LOGOUT_WORKER_TASKQUEUE', '')

GenespotRE/settings_unit_test.py

@@ -291,12 +291,7 @@
 #################################
 
 LOGIN_EXPIRATION_HOURS = 24
-FAKE_DBGAP_AUTHENTICATION_LIST_FILENAME = os.environ.get('FAKE_DBGAP_AUTHENTICATION_LIST_FILENAME', '') # This should be removed in favour of putting the change in .env files
-DBGAP_AUTHENTICATION_LIST_FILENAME      = os.environ.get('DBGAP_AUTHENTICATION_LIST_FILENAME', '')
-DBGAP_AUTHENTICATION_LIST_BUCKET        = os.environ.get('DBGAP_AUTHENTICATION_LIST_BUCKET', '')
 GOOGLE_GROUP_ADMIN                      = os.environ.get('GOOGLE_GROUP_ADMIN', '')
 SUPERADMIN_FOR_REPORTS                  = os.environ.get('SUPERADMIN_FOR_REPORTS', '')
 OPEN_ACL_GOOGLE_GROUP                   = os.environ.get('OPEN_ACL_GOOGLE_GROUP', '')
-ERA_LOGIN_URL                           = os.environ.get('ERA_LOGIN_URL', '')
-SAML_FOLDER                             = os.environ.get('SAML_FOLDER')
 

GenespotRE/views.py

@@ -53,7 +53,6 @@
 debug = settings.DEBUG
 logger = logging.getLogger('main_logger')
 
-ERA_LOGIN_URL = settings.ERA_LOGIN_URL
 OPEN_ACL_GOOGLE_GROUP = settings.OPEN_ACL_GOOGLE_GROUP
 BQ_ATTEMPT_MAX = 10
 WEBAPP_LOGIN_LOG_NAME = settings.WEBAPP_LOGIN_LOG_NAME
@@ -158,8 +157,7 @@ def user_detail(request, user_id):
         return render(request, 'GenespotRE/user_detail.html',
                       {'request': request,
                        'user': user,
-                       'user_details': user_details,
-                       'ERA_LOGIN_URL': settings.ERA_LOGIN_URL
+                       'user_details': user_details
                        })
     else:
         return render(request, '403.html')

README.md

@@ -1,6 +1,6 @@
 # ISB-CGC Web Application
 
-The ISB-CGC Web Application provides a GUI for browsing, curating, and analyzing TCGA, TARGET, and CCLE data. It is built in Django 1.11 (Python 2.7) with MySQL 5.7, and is deployed on Google AppEngine Flex.
+The ISB-CGC Web Application provides a GUI for browsing, curating, and analyzing TCGA, TARGET, and CCLE data. It is built in Django 1.11 (Python 3) with MySQL 5.7, and is deployed on Google AppEngine Flex.
 
 # Installation Instructions For Local Development
 
@@ -10,12 +10,12 @@ The system uses [Vagrant](https://www.vagrantup.com/) to setup a consistent, pla
  * [Oracle VirtualBox](https://www.virtualbox.org/wiki/Downloads)<br>*If you are on Windows 8 or above, you will need to make sure you have version 5.0.9 or above to support the network interface. This may involve downloading a [test build](https://www.virtualbox.org/wiki/Testbuilds)*
  * [PyCharm Pro](https://www.jetbrains.com/pycharm/) (Recommended)
 
-From there simply perform these steps.
+From there perform the following steps...
 
  1. Once you've installed PyCharm and cloned the repositories, create a directory within the `PycharmProjects` directory (the parent directory of your repositories) called `secure_files/`.
  2. Copy the `sample.env` file to a file named `.env` in `secure_files/`
  3. Fill out the `.env` file with the proper values
-   * For most development environments, `MYSQL_ROOT_PASSWORD` and `DATABASE_PASSWORD` can be the same, and `DATABASE_USER` can be `root`
+   * For most **development** environments, `MYSQL_ROOT_PASSWORD` and `DATABASE_PASSWORD` can be the same, and `DATABASE_USER` can be `root`
    * `GCLOUD_PROJECT_ID` is available after creating a project in the [Google Cloud Dashboard](https://console.developers.google.com/)
    * `GOOGLE_CLIENT_ID` and `GOOGLE_CLIENT_SECRET` can also be obtained in the Google Cloud Dashboard by going to API & Auth > Credentials > Add New > OAuth 2.0 Client > Web Application
    * Be sure when developing locally that you have 127.0.0.1 in the list of allowed domains for the OAuth 2.0 key
@@ -54,24 +54,28 @@ You will also need to set the *shell/python-su.sh* file to be executable. You ca
 To run your server in PyCharm:
 
  1. Make sure your Vagrant machine is running by going to **Tools > Vagrant > Up**
- 2. Click on the Run or Debug icons in the toolbar (upper-right corner of the PyCharm GUI)
-
-Your server will start and the PyCharm console should show all the logs and output from the system. If you are running in debug, you can also use breakpoints to stop the execution and examine variables and code as it runs.
+  * If this is the first time you've built the VM, it can be time consuming.
+  * Our VMs are currently running Ubuntu 16 LTS, which is what the app deploys under as well.
+ 2. Once the VM has built, click on the Run or Debug icons in the toolbar (upper-right corner of the PyCharm GUI)
+  * Your server will start and the PyCharm console should show all the logs and output from the system. 
+  * If you are running in debug, you can also use breakpoints to stop the execution and examine variables and code as it runs.
 
 ## Adding Python Dependencies
 
 To add Python Libraries or Dependencies, you should add them to the requirements.txt file and they will automatically be pulled down when a new developer starts the system.
+ * Double-check any new libraries to make sure they don't introduce conflicts.
+ * Note that sometimes a library will function fine on a local build but fail on the deployment, so always test on the mvm deployment as soon as your PR has been merged into master.
 
-To update your existing python dependencies because of a change or to pull down additional libraries you need, SSH into the virtual machine and run `pip install`. Through PyCharm, you can take the following steps.
+To update your existing python dependencies because of a change, or to pull down additional libraries you need, SSH into the virtual machine and run `pip3 install`. Through PyCharm, you can take the following steps:
 
  1. Click **Tools > Start SSH session...**
  2. Select the Vagrant VM Connection you set up
- 3. Type `cd www; sudo pip install -r requirements.txt --upgrade -t lib/`
+ 3. Type `cd www; sudo pip3 install -r requirements.txt --upgrade -t lib/`
 
-Or from the command line, you can do this by doing the following
+Or from the command line, you can do this by doing the following:
 
  1. Open a terminal in the project directory
  2. Type `vagrant ssh` to login to the virtual machine
  3. Change directory to the `www` directory (`/home/vagrant/www/` is the full path)
- 4. Run `pip install -r requirements.txt --upgrade -t lib/`
+ 4. Run `pip3 install -r requirements.txt --upgrade -t lib/`
 

requirements.txt

@@ -1,5 +1,5 @@
 requests==2.20.0
-django==1.11.20
+django==1.11.21
 PyYAML==4.2b1
 mysqlclient==1.3.13
 WebOb==1.5.0

shell/gcloud_authenticate.sh

@@ -1,4 +1,4 @@
-echo $PROD_PRIVATE_KEY_FOR_V2 | base64 --decode --ignore-garbage > deployment.key.json
+echo $DEPLOYMENT_KEY_ISB_CGC | base64 --decode --ignore-garbage > deployment.key.json
 
 /home/circleci/${CIRCLE_PROJECT_REPONAME}/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file deployment.key.json
 /home/circleci/${CIRCLE_PROJECT_REPONAME}/google-cloud-sdk/bin/gcloud config set account $GAE_CLIENT_EMAIL

static/js/helpers/vis_helpers.js

@@ -26,20 +26,20 @@ define(['jquery'], function($) {
                 (val.match(/^-?\d*\.?\d+$/) !== null)
             ));
         },
-        get_min_max: function(data, selector) {
+        get_min_max: function(data, selector, include_nan) { //include_nan: true if to handle NaN data as 0
             var self=this;
             var min = d3.min(data, function(d) {
                 if (self.isValidNumber(d[selector])) {
                     return parseFloat(d[selector]);
                 } else {
-                    return undefined;
+                    return include_nan ? 0 : undefined;
                 }
             });
             var max = d3.max(data, function(d) {
                 if (self.isValidNumber(d[selector])) {
                     return parseFloat(d[selector]);
                 } else {
-                    return undefined;
+                    return include_nan ? 0 : undefined;
                 }
             });
             min = isNaN(min) ? 0 : min;

static/js/variables/variables.js

@@ -246,6 +246,7 @@ require([
             $.ajax({
                 type: 'POST',
                 url : BASE_URL + '/variables/save',
+                contentType: "application/json",
                 data: JSON.stringify({name : name, variables : variable_list}),
                 beforeSend: function(xhr){xhr.setRequestHeader("X-CSRFToken", csrftoken);},
                 success: function (data) {
@@ -313,6 +314,7 @@ require([
             $.ajax({
                 type : 'POST',
                 url  : BASE_URL + '/variables/' + variable_id + '/update',
+                contentType: "application/json",
                 data : JSON.stringify({name : name, variables : variable_list}),
                 beforeSend: function(xhr){xhr.setRequestHeader("X-CSRFToken", csrftoken);},
                 success: function (data) {
@@ -353,6 +355,7 @@ require([
             $.ajax({
                 type : 'POST',
                 url  : BASE_URL + '/workbooks/' + workbook_id + '/worksheets/' + worksheet_id + '/variables/edit',
+                contentType: "application/json",
                 data : JSON.stringify({name : name, variables : variable_list}),
                 beforeSend: function(xhr){xhr.setRequestHeader("X-CSRFToken", csrftoken);},
                 success: function (data) {
@@ -379,6 +382,7 @@ require([
             $.ajax({
                 type: 'POST',
                 url : BASE_URL + '/variables/save',
+                contentType: "application/json",
                 data: JSON.stringify({name : name, variables : variable_list}),
                 beforeSend: function(xhr){xhr.setRequestHeader("X-CSRFToken", csrftoken);},
                 success: function (data) {

static/js/visualizations/createBarGraph.js

@@ -30,17 +30,6 @@ define(['jquery', 'd3', 'd3tip', 'd3textwrap', 'vizhelpers', 'underscore'],
         // The samples found in the selected value buckets; this is used to produce the JSON which
         // is submitted by the form
         var selectedSamples = null;
-
-        // If you want to override the tip coming in from the create call,
-        // do it here
-        var tip = d3tip()
-            .attr('class', 'd3-tip')
-            .direction('n')
-            .offset([0, 0])
-            .html(function (d) {
-                return d.value + ': ' + d.count;
-            });
-
         var selex_active = false;
         var zoom_status = {
             translation: null,
@@ -49,40 +38,52 @@ define(['jquery', 'd3', 'd3tip', 'd3textwrap', 'vizhelpers', 'underscore'],
 
         return {
 
-            dataCounts: function (data, x_attr) {
-                var counts = {};
+            dataCounts: function (data, x_attr, bySample) {
+                var val_set = {};
                 var results = [];
 
-                for (var i = 0; i < data.length; i++) {
-                    var val = data[i][x_attr];
-                    if (!counts[val]) {
-                        counts[val] = 0;
+                data.map(function(d){
+                    var val = d[x_attr];
+                    var sample_id = d['sample_id'];
+                    var case_id = d['case_id'];
+                    if (!val_set[val]) {
+                        val_set[val] = {};
                     }
-                    counts[val] += 1;
 
+                    if(bySample){
+                        val_set[val][sample_id] = 1;
+                    }
+                    else if(!val_set[val][case_id]){
+                        val_set[val][case_id] = 1
+                    }
                     if (!sampleSet[val]) {
-                        sampleSet[val] = {
-                            samples: {},
-                            cases: new Set([])
-                        };
+                        sampleSet[val] = {};
                     }
-
-                    sampleSet[val].samples['{' + data[i]['sample_id'] + '}{' + data[i]['case_id'] + '}'] = {
-                        sample: data[i]['sample_id'],
-                        case: data[i]['case_id'],
-                        project: data[i]['project']
+                    sampleSet[val]['{' + sample_id + '}{' + case_id + '}'] = {
+                        sample: sample_id,
+                        case: case_id,
+                        project: d['project']
                     };
-                    sampleSet[val].cases.add(data[i]['case_id']);
-                }
+                });
 
-                for (var key in counts) {
-                    results.push({'value': key, 'count': counts[key]});
+                for (var key in val_set) {
+                    results.push({'value': key, 'count': Object.keys(val_set[key]).length});
                 }
 
                 return results;
             },
-            createBarGraph: function (svg, raw_Data, width, height, bar_width, x_attr, xLabel, margin, legend) {
-                var data = this.dataCounts(raw_Data, x_attr);
+            createBarGraph: function (svg, raw_Data, width, height, bar_width, x_attr, xLabel, margin, bySample) {
+                // If you want to override the tip coming in from the create call,
+                // do it here
+                var tip = d3tip()
+                    .attr('class', 'd3-tip')
+                    .direction('n')
+                    .offset([0, 0])
+                    .html(function (d) {
+                        return d.value + ': ' + d.count + (bySample ? ' sample' : ' case') + (d.count > 1 ? 's':'');
+                    });
+
+                var data = this.dataCounts(raw_Data, x_attr, bySample);
                 var plot_width = (bar_width + 5) * data.length;
                 var view_plot_width = plot_width < width - margin.left - margin.right ? width - margin.left - margin.right : plot_width;
 
@@ -277,7 +278,7 @@ define(['jquery', 'd3', 'd3tip', 'd3textwrap', 'vizhelpers', 'underscore'],
                 svg.append('text')
                     .attr('class', 'y label axis-label')
                     .attr('text-anchor', 'middle')
-                    .text('Number of Samples')
+                    .text('Number of '+ (bySample ? 'Samples' : 'Cases'))
                     .attr('transform', 'rotate(-90) translate(' + (-height - margin.top + margin.bottom) / 2 + ', 20)');
 
                 var check_selection_state = function (obj) {
@@ -323,9 +324,9 @@ define(['jquery', 'd3', 'd3tip', 'd3textwrap', 'vizhelpers', 'underscore'],
                         var case_set = {};
                         selectedSamples = {};
                         _.each(Object.keys(selectedValues), function (val) {
-                            _.each(Object.keys(sampleSet[val]['samples']), function (sample) {
-                                selectedSamples[sample] = sampleSet[val]['samples'][sample];
-                                case_set[sampleSet[val]['samples'][sample]['case']] = 1;
+                            _.each(Object.keys(sampleSet[val]), function (sample) {
+                                selectedSamples[sample] = sampleSet[val][sample];
+                                case_set[sampleSet[val][sample]['case']] = 1;
                             });
                         });
 
@@ -338,7 +339,6 @@ define(['jquery', 'd3', 'd3tip', 'd3textwrap', 'vizhelpers', 'underscore'],
                         var leftVal = Math.min((x3(extent[1]) + 20), (width - $('.worksheet.active .save-cohort-card').width()));
                         $('.worksheet.active .save-cohort-card').show()
                             .attr('style', 'position:relative; top: -' + height + 'px; left:' + leftVal + 'px;');
-
                     }
 
                 }