Add Mock Datastores (#329)

* Added a mock Datastore This returns the mocked results of all the functions without making any requests to the actual Databases * Hook up MockDataStore to main function * Minor syntax error * Eslint * Added mocks for Apache Drill * eslint * Addressed 🐐 comments * Renamed 'DataStoreMock' -> 'datastoremock' * Added tests * Use 'connection.mock' instead of environment var for recognizing mocks * Added separate mock connections. * Updated tests * Fixed tests * Clear settings on test finish * eslint * Reset 'mock' on test completion * Added mock storage * Use mock-storage for mock tests * Use mock storage for Heroku App. * Eslint * removed extra code * Addressed comment * Addressed comments * 'undefined' folder issue * Delete environment var if not set initially
plotly · Jan 12, 2018 · 5f0cb2a · 5f0cb2a
1 parent 6d40abf
commit 5f0cb2a
Show file tree

Hide file tree

Showing 6 changed files with 446 additions and 1 deletion.
diff --git a/Procfile b/Procfile
@@ -1 +1 @@
-web: PLOTLY_CONNECTOR_PORT=$PORT PLOTLY_CONNECTOR_STORAGE_PATH="/app/sample-storage" PLOTLY_CONNECTOR_AUTH_ENABLED=false node ./dist/headless-bundle.js
+web: PLOTLY_CONNECTOR_PORT=$PORT PLOTLY_CONNECTOR_STORAGE_PATH="/app/mock-storage" PLOTLY_CONNECTOR_AUTH_ENABLED=false node ./dist/headless-bundle.js
diff --git a/backend/persistent/datastores/Datastores.js b/backend/persistent/datastores/Datastores.js
@@ -5,6 +5,7 @@ import * as ApacheDrill from './ApacheDrill';
 import * as IbmDb2 from './ibmdb2';
 import * as ApacheLivy from './livy';
 import * as ApacheImpala from './impala';
+import * as DatastoreMock from './datastoremock';
 
 /*
  * Switchboard to all of the different types of connections
@@ -26,6 +27,11 @@ import * as ApacheImpala from './impala';
  */
 
 function getDatastoreClient(connection) {
+    // handle test mode:
+    if (connection.mock) {
+        return DatastoreMock;
+    }
+
     const {dialect} = connection;
     if (dialect === 'elasticsearch') {
         return Elasticsearch;

diff --git a/backend/persistent/datastores/datastoremock.js b/backend/persistent/datastores/datastoremock.js
@@ -0,0 +1,131 @@
+/*
+ * Mock functions return the hardcoded values without hitting any DB
+ */
+
+export function connect() {
+  return new Promise();
+}
+
+export function tables() {
+    return Promise.resolve(['TABLE_A', 'TABLE_B', 'TABLE_C', 'TABLE_D']);
+}
+
+export function query(queryString) {
+    return new Promise((resolve, reject) => {
+        if (queryString === 'ERROR') {
+            reject(new Error('Syntax Error in Query'));
+        } else {
+            resolve({
+                'columnnames': ['COLUMN_A', 'COLUMN_B', 'COLUMN_C'],
+                'rows': [
+                    ['ROW_1', '1.112', '12'],
+                    ['ROW_2', '2.2', '98'],
+                    ['ROW_3', '3.12', '62']
+                ]
+            });
+        }
+    });
+}
+
+export function files() {
+    return Promise.resolve([
+        {
+            'Key': 'A.csv',
+            'LastModified': '2016-10-09T17:29:49.000Z',
+            'ETag': '\'635633cb59c369da25fdf7bd6cc8de62\'',
+            'Size': 151650,
+            'StorageClass': 'STANDARD',
+            'Owner': {
+                'DisplayName': 'chris',
+                'ID': '655b5b49d59fe8784105e397058bf0f410579195145a701c03b55f10920bc67a'
+            }
+         },
+         {
+             'Key': 'B.csv',
+             'LastModified': '2016-10-09T17:29:49.000Z',
+             'ETag': '\'635633cb59c369da25fdf7bd6cc8de62\'',
+             'Size': 151650,
+             'StorageClass': 'STANDARD',
+             'Owner': {
+                 'DisplayName': 'chris',
+                 'ID': '655b5b49d59fe8784105e397058bf0f410579195145a701c03b55f10920bc67a'
+             }
+         }
+    ]);
+}
+
+export function elasticsearchMappings() {
+    return Promise.resolve({
+        'test-mappings': {
+            'mappings': {
+                'TABLE_A': {
+                    'properties': {
+                        'COLUMN_A': {'type': 'string'},
+                        'COLUMN_B': {'type': 'float'},
+                        'COLUMN_C': {'type': 'integer'}
+                    }
+                },
+                'TABLE_B': {
+                    'properties': {
+                        'COLUMN_M': {'type': 'string'},
+                        'COLUMN_N': {'type': 'float'},
+                        'COLUMN_O': {'type': 'integer'}
+                    }
+                }
+            }
+        }
+    });
+}
+
+export function storage() {
+    return Promise.resolve([
+        {
+            'name': 's3',
+            'config': {
+                'type': 'file',
+                'enabled': true,
+                'connection': 's3a://plotly-s3-connector-test',
+                'config': {
+                    'fs.s3a.access.key': 'ABCD',
+                    'fs.s3a.secret.key': 'MNOP'
+                },
+                'workspaces': {
+                    'root': {
+                        'location': '/',
+                        'writable': true,
+                        'defaultInputFormat': null
+                    }
+                },
+                'formats': {'parquet': {'type': 'parquet'}}
+            }
+        }
+    ]);
+}
+
+// This is for Apache Drill:
+export function listS3Files() {
+    return Promise.resolve([
+        {
+            'Key': 'A.parquet',
+            'LastModified': '2016-10-09T17:29:49.000Z',
+            'ETag': '\'635633cb59c369da25fdf7bd6cc8de62\'',
+            'Size': 151650,
+            'StorageClass': 'STANDARD',
+            'Owner': {
+                'DisplayName': 'chris',
+                'ID': '655b5b49d59fe8784105e397058bf0f410579195145a701c03b55f10920bc67a'
+            }
+         },
+         {
+             'Key': 'B.parquet',
+             'LastModified': '2016-10-09T17:29:49.000Z',
+             'ETag': '\'635633cb59c369da25fdf7bd6cc8de62\'',
+             'Size': 151650,
+             'StorageClass': 'STANDARD',
+             'Owner': {
+                 'DisplayName': 'chris',
+                 'ID': '655b5b49d59fe8784105e397058bf0f410579195145a701c03b55f10920bc67a'
+             }
+         }
+    ]);
+}
diff --git a/mock-storage/connections.yaml b/mock-storage/connections.yaml
@@ -0,0 +1,108 @@
+-
+    dialect: postgres
+    id: postgres-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    username: masteruser
+    password: connecttoplotly
+    host: readonly-test-postgres.cwwxgcilxwxw.us-west-2.rds.amazonaws.com
+    port: 5432
+    database: plotly_datasets
+    mock: true
+
+-
+    dialect: mysql
+    id: mysql-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    username: masteruser
+    password: connecttoplotly
+    host: readonly-test-mysql.cwwxgcilxwxw.us-west-2.rds.amazonaws.com
+    port: 3306
+    database: plotly_datasets
+    mock: true
+
+-
+    dialect: mariadb
+    id: mariadb-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    username: masteruser
+    password: connecttoplotly
+    host: readonly-test-mariadb.cwwxgcilxwxw.us-west-2.rds.amazonaws.com
+    port: 3306
+    database: plotly_datasets
+    mock: true
+
+-
+    dialect: redshift
+    id: redshift-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    username: plotly
+    password: Qmbdf#3DU]pP8a=CKTK}
+    host: sql-connector-test.cfiaqtidutxu.us-east-1.redshift.amazonaws.com
+    port: 5439
+    database: plotly_datasets
+    mock: true
+
+-
+    dialect: mssql
+    id: mssql-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    username: masteruser
+    password: connecttoplotly
+    host: test-mssql.cwwxgcilxwxw.us-west-2.rds.amazonaws.com
+    port: 1433
+    database: plotly_datasets
+    mock: true
+
+-
+    dialect: sqlite
+    id: sqlite-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    storage: /app/test/backend/plotly_datasets.db
+    mock: true
+
+-
+    dialect: elasticsearch
+    id: elasticsearch-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    host: https://67a7441549120daa2dbeef8ac4f5bb2e.us-east-1.aws.found.io
+    port: 9243
+    mock: true
+
+-
+    dialect: s3
+    id: s3-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    bucket: plotly-s3-connector-test
+    accessKeyId: AKIAIMHMSHTGARJYSKMQ
+    secretAccessKey: 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
+    mock: true
+
+-
+    dialect: apache drill
+    id: apache drill-189ebfb4-e1b4-446c-9b83-b326875fa2d8
+    host: http://ec2-35-164-71-216.us-west-2.compute.amazonaws.com
+    port: 8047
+    bucket: plotly-s3-connector-test
+    accessKeyId: AKIAIMHMSHTGARJYSKMQ
+    secretAccessKey: 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
+    mock: true
+
+-
+    dialect: ibm db2
+    id: ibm db2-f51f57b6-d00b-478a-b502-8e201ae70e74
+    username: db2user1
+    password: w8wfy99DvEmgkBsE
+    database: plotly
+    port: 50000
+    host: 35.184.35.183
+    mock: true
+
+-
+    dialect: apache spark
+    id: apache spark-7b7bfa1e-cdfa-4001-b326-e0e170667021
+    timeout: 180
+    database: plotly
+    port: 8998
+    host: 104.154.141.189
+    mock: true
+
+-
+    dialect: apache impala
+    id: apache impala-159e0b47-0428-4c9e-b4b9-8201b86f8ca2
+    timeout: 180
+    database: plotly
+    port: 21000
+    host: 35.184.155.127
+    mock: true
diff --git a/package.json b/package.json
@@ -32,6 +32,7 @@
     "test-unit-oauth2": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/routes.oauth2.spec.js",
     "test-unit-scheduler": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/QueryScheduler.spec.js",
     "test-unit-routes": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/routes.spec.js",
+    "test-unit-mock": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.mock.spec.js",
     "pack": "cross-env NODE_ENV=production electron-builder --publish=never",
     "package": "cross-env NODE_ENV=production node -r babel-register package.js",
     "package-all": "yarn run package -- --all",
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		web: PLOTLY_CONNECTOR_PORT=$PORT PLOTLY_CONNECTOR_STORAGE_PATH="/app/sample-storage" PLOTLY_CONNECTOR_AUTH_ENABLED=false node ./dist/headless-bundle.js
		web: PLOTLY_CONNECTOR_PORT=$PORT PLOTLY_CONNECTOR_STORAGE_PATH="/app/mock-storage" PLOTLY_CONNECTOR_AUTH_ENABLED=false node ./dist/headless-bundle.js