Merge branch 'develop' into 'master'

Update master

See merge request back-end/elastalert!28

.gitlab-ci.yml

@@ -60,12 +60,12 @@ deploy:docker:
   script:
     - sh scripts/replace_templates.sh ${CI_JOB_TOKEN}
     - docker info
-    - echo $BITSENSOR_PASSWORD | docker login -u $BITSENSOR_USERNAME --password-stdin artifacts.bitsensor.io:1443
+    - echo $BITSENSOR_PASSWORD | docker login -u $BITSENSOR_USERNAME --password-stdin docker.bitsensor.io
     - docker build -t elastalert .
-    - docker tag elastalert artifacts.bitsensor.io:1443/elastalert:latest
-    - docker tag elastalert artifacts.bitsensor.io:1443/elastalert:$(git describe --abbrev=0)
-    - docker push artifacts.bitsensor.io:1443/elastalert:latest
-    - docker push artifacts.bitsensor.io:1443/elastalert:$(git describe --abbrev=0)
+    - docker tag elastalert docker.bitsensor.io/elastalert:latest
+    - docker tag elastalert docker.bitsensor.io/elastalert:$(git describe --abbrev=0)
+    - docker push docker.bitsensor.io/elastalert:latest
+    - docker push docker.bitsensor.io/elastalert:$(git describe --abbrev=0)
   only:
     - tags
   tags:
@@ -76,10 +76,10 @@ deploy:docker:snapshot:
   script:
     - sh scripts/replace_templates.sh ${CI_JOB_TOKEN}
     - docker info
-    - echo $BITSENSOR_PASSWORD | docker login -u $BITSENSOR_USERNAME --password-stdin artifacts.bitsensor.io:1443
+    - echo $BITSENSOR_PASSWORD | docker login -u $BITSENSOR_USERNAME --password-stdin docker.bitsensor.io
     - docker build -t elastalert .
-    - docker tag elastalert artifacts.bitsensor.io:1443/elastalert:snapshot
-    - docker push artifacts.bitsensor.io:1443/elastalert:snapshot
+    - docker tag elastalert docker.bitsensor.io/elastalert:snapshot
+    - docker push docker.bitsensor.io/elastalert:snapshot
   only:
     - develop
   tags:

Dockerfile

@@ -1,5 +1,5 @@
 FROM alpine:latest as py-ea
-ARG ELASTALERT_VERSION=v0.1.33
+ARG ELASTALERT_VERSION=v0.1.36
 ENV ELASTALERT_VERSION=${ELASTALERT_VERSION}
 # URL from which to download Elastalert.
 ARG ELASTALERT_URL=https://github.com/Yelp/elastalert/archive/$ELASTALERT_VERSION.zip
@@ -26,12 +26,10 @@ RUN sed -i 's/jira>=1.0.10/jira>=1.0.10,<1.0.15/g' setup.py && \
 
 FROM node:alpine
 LABEL maintainer="BitSensor <dev@bitsensor.io>"
-# Set this environment variable to True to set timezone on container start.
-ENV SET_CONTAINER_TIMEZONE False
-# Default container timezone as found under the directory /usr/share/zoneinfo/.
-ENV CONTAINER_TIMEZONE Etc/UTC
+# Set timezone for this container
+ENV TZ Etc/UTC
 
-RUN apk add --update --no-cache curl tzdata python2 make
+RUN apk add --update --no-cache curl tzdata python2 make libmagic
 
 COPY --from=py-ea /usr/lib/python2.7/site-packages /usr/lib/python2.7/site-packages
 COPY --from=py-ea /opt/elastalert /opt/elastalert

README.md

@@ -111,7 +111,10 @@ You can use the following config options:
   "dataPath": { // The path to a folder that the server can use to store data and temporary files.
     "relative": true, // Whether to use a path relative to the `elastalertPath` folder.
     "path": "/server_data" // The path to the data folder.
-  }
+  },
+  "es_host": "elastalert", // For getting metadata and field mappings, connect to this ES server
+  "es_port": 9200, // Port for above
+  "writeback_index": "elastalert_status" // Writeback index to examine for /metadata endpoint
 }
 ```
 
@@ -200,11 +203,25 @@ This server exposes the following REST API's:
           "days": "1"
           
           // Whether to send real alerts
-          "alert": false
+          "alert": false,
+
+          // Return results in structured JSON
+          "format": "json",
+
+          // Limit returned results to this amount
+          "maxResults": 1000
         }
       }
       ``` 
 
+- **GET `/metadata/:type`**
+
+    Returns metadata from elasticsearch related to elasalert's state. `:type` should be one of: elastalert_status, elastalert, elastalert_error, or silence. See [docs about the elastalert metadata index](https://elastalert.readthedocs.io/en/latest/elastalert_status.html).
+
+- **GET `/mapping/:index`**
+
+    Returns field mapping from elasticsearch for a given index. 
+
 - **[WIP] GET `/config`**
 
     Gets the ElastAlert configuration from `config.yaml` in `elastalertPath` (from the config).

config/config.json

@@ -12,5 +12,8 @@
   "templatesPath": {
     "relative": true,
     "path": "/rule_templates"
-  }
-}
+  },
+  "es_host": "elastalert",
+  "es_port": 9200,
+  "writeback_index": "elastalert_status"
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bitsensor/elastalert",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "description": "A server that runs ElastAlert and exposes REST API's for manipulating rules and alerts.",
   "license": "MIT",
   "main": "index.js",
@@ -11,7 +11,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://git.bitsensor.io/back-end/elastalert.git"
+    "url": "git+https://github.com/bitsensor/elastalert.git"
   },
   "directories": {
     "lib": "./lib",
@@ -23,6 +23,8 @@
     "babel-register": "^6.14.0",
     "body-parser": "^1.15.2",
     "bunyan": "^1.8.1",
+    "cors": "^2.8.4",
+    "elasticsearch": "^15.1.1",
     "express": "^4.14.0",
     "fs-extra": "^5.0.0",
     "joi": "^13.1.2",

src/common/config/schema.js

@@ -3,6 +3,9 @@ import Joi from 'joi';
 
 const schema = Joi.object().keys({
   'appName': Joi.string().default('elastalert-server'),
+  'es_host': Joi.string().default('elastalert'),
+  'es_port': Joi.number().default(9200),
+  'writeback_index': Joi.string().default('elastalert_status'),
   'port': Joi.number().default(3030),
   'elastalertPath': Joi.string().default('/opt/elastalert'),
   'rulesPath': Joi.object().keys({

src/common/elasticsearch_client.js

@@ -0,0 +1,9 @@
+import elasticsearch from 'elasticsearch';
+import config from './config';
+
+export function getClient() {
+  var client = new elasticsearch.Client({
+    hosts: [ `http://${config.get('es_host')}:${config.get('es_port')}`]
+  });
+  return client;
+}

src/controllers/test/index.js

@@ -33,6 +33,15 @@ export default class TestController {
 
           processOptions.push('-m', 'elastalert.test_rule', '--config', 'config.yaml', tempFilePath, '--days', options.days);
 
+          if (options.format === 'json') {
+            processOptions.push('--formatted-output');
+          }
+
+          if (options.maxResults > 0) {
+            processOptions.push('--max-query-size');
+            processOptions.push(options.maxResults);
+          }
+
           if (options.alert) {
             processOptions.push('--alert');
           }
@@ -61,7 +70,12 @@ export default class TestController {
 
             testProcess.on('exit', function (statusCode) {
               if (statusCode === 0) {
-                resolve(stdoutLines.join('\n'));
+                if (options.format === 'json') {
+                  resolve(stdoutLines.join(''));
+                }
+                else {
+                  resolve(stdoutLines.join('\n'));
+                }
               } else {
                 reject(stderrLines.join('\n'));
                 logger.error(stderrLines.join('\n'));

src/elastalert_server.js

@@ -9,6 +9,7 @@ import ProcessController from './controllers/process';
 import RulesController from './controllers/rules';
 import TemplatesController from './controllers/templates';
 import TestController from './controllers/test';
+import cors from 'cors';
 
 let logger = new Logger('Server');
 
@@ -58,6 +59,7 @@ export default class ElastalertServer {
     // Start the server when the config is loaded
     config.ready(function () {
       try {
+        self._express.use(cors());
         self._express.use(bodyParser.json());
         self._express.use(bodyParser.urlencoded({ extended: true }));
         self._setupRouter();

src/handlers/mapping/get.js

@@ -0,0 +1,20 @@
+import { getClient } from '../../common/elasticsearch_client';
+
+export default function metadataHandler(request, response) {
+  /**
+   * @type {ElastalertServer}
+   */
+
+  var client = getClient();
+
+  client.indices.getMapping({
+    index: request.params.index
+  }).then(function(resp) {
+    response.send(resp);
+  }, function(err) {
+    response.send({
+      error: err
+    });
+  });
+
+}

src/handlers/metadata/get.js

@@ -0,0 +1,42 @@
+import config from '../../common/config';
+import { getClient } from '../../common/elasticsearch_client';
+
+
+function getQueryString(request) {
+  if (request.params.type === 'elastalert_error') {
+    return '*:*';
+  }
+  else {
+    return `rule_name:${request.query.rule_name || '*'}`;
+  }
+}
+
+export default function metadataHandler(request, response) {
+  /**
+   * @type {ElastalertServer}
+   */
+  var client = getClient();
+
+  client.search({
+    index: config.get('writeback_index'),
+    type: request.params.type,
+    body: {
+      from : request.query.from || 0, 
+      size : request.query.size || 100,
+      query: {
+        query_string: {
+          query: getQueryString(request)
+        }
+      },
+      sort: [{ '@timestamp': { order: 'desc' } }]
+    }
+  }).then(function(resp) {
+    resp.hits.hits = resp.hits.hits.map(h => h._source);
+    response.send(resp.hits);
+  }, function(err) {
+    response.send({
+      error: err
+    });
+  });
+
+}

src/handlers/templates/id/post.js

@@ -26,8 +26,8 @@ export default function templatePostHandler(request, response) {
         });
     })
     .catch(function (error) {
-      if (error.error === 'ruleNotFound') {
-        server.templatesController.createRule(request.params.id, body)
+      if (error.error === 'templateNotFound') {
+        server.templatesController.createTemplate(request.params.id, body)
           .then(function () {
             logger.sendSuccessful();
             response.send({

src/handlers/test/post.js

@@ -8,7 +8,9 @@ let logger = new RouteLogger('/test', 'POST');
 const optionsSchema = Joi.object().keys({
   testType: Joi.string().valid('all', 'schemaOnly', 'countOnly').default('all'),
   days: Joi.number().min(1).default(1),
-  alert: Joi.boolean().default(false)
+  alert: Joi.boolean().default(false),
+  format: Joi.string().default(''),
+  maxResults: Joi.number().default(0)
 }).default();
 
 function analyzeRequest(request) {

src/routes/routes.js

@@ -14,6 +14,8 @@ import templateDeleteHandler from '../handlers/templates/id/delete';
 import testPostHandler from '../handlers/test/post';
 import configGetHandler from '../handlers/config/get';
 import configPostHandler from '../handlers/config/post';
+import metadataHandler from '../handlers/metadata/get';
+import mappingHandler from '../handlers/mapping/get';
 
 /**
  * A server route.
@@ -74,6 +76,16 @@ let routes = [
     path: 'download',
     method: ['POST'],
     handler: [downloadRulesHandler]
+  },
+  {
+    path: 'metadata/:type',
+    method: ['GET'],
+    handler: [metadataHandler]
+  },
+  {
+    path: 'mapping/:index',
+    method: ['GET'],
+    handler: [mappingHandler]
   }
 ];