SNO-179-allow-devserver-set-es-config-dir

src/snovault/dev_servers.py

@@ -60,6 +60,13 @@ def main():
     parser.add_argument('--init', action="store_true", help="Init database")
     parser.add_argument('--load', action="store_true", help="Load test set")
     parser.add_argument('--datadir', default='/tmp/snovault', help="path to datadir")
+    parser.add_argument(
+        '-e',
+        '--es-version',
+        default=5,
+        type=int,
+        help='Override defult elasticsearch_fixture es version'
+    )
     args = parser.parse_args()
 
     appsettings = get_appsettings(args.config_uri, name='app')
@@ -77,17 +84,19 @@ def main():
     from snovault.elasticsearch import create_mapping
     datadir = os.path.abspath(args.datadir)
     pgdata = os.path.join(datadir, 'pgdata')
-    esdata = os.path.join(datadir, 'esdata')
     redisdata = os.path.join(datadir, 'redisdata')
     if args.clear:
-        for dirname in [pgdata, esdata, redisdata]:
-            if os.path.exists(dirname):
-                shutil.rmtree(dirname)
+        if os.path.exists(args.datadir):
+            shutil.rmtree(args.datadir)
     if args.init:
         postgresql_fixture.initdb(pgdata, echo=True)
-
     postgres = postgresql_fixture.server_process(pgdata, echo=True)
-    elasticsearch = elasticsearch_fixture.server_process(esdata, echo=True)
+    elasticsearch = elasticsearch_fixture.server_process(
+        datadir,
+        clear=args.clear,
+        echo=True,
+        version=args.es_version,
+    )
     nginx = nginx_server_process(echo=True)
     redis_config_path = redis_storage_fixture.initdb(redisdata, local_storage_port, echo=True)
     redis = redis_storage_fixture.server_process(redis_config_path, local_storage_port, local_storage_redis_index, echo=True)

src/snovault/tests/elasticsearch_fixture.py

@@ -1,4 +1,5 @@
 import os.path
+import shutil
 import sys
 from time import sleep
 try:
@@ -7,34 +8,98 @@
     import subprocess
 
 
-def server_process(datadir, host='127.0.0.1', port=9201, prefix='', echo=False):
+# This is required for local osx testing, circle version is sepearte
+_OSX_ES_VERSION = 5
+# These are set in .circleci/config
+_CIRCLE_BUILD = os.environ.get('BASH_ENV') == '/home/circleci/.bashrc'
+_CIRCLE_ES_VERSION = os.environ.get('ES_MAJOR_VERSION', str(_OSX_ES_VERSION))
+# Encoded configuraion repo
+_ENCD_CONFIG_DIR = os.environ.get('ENCD_CONFIG_DIR')
+
+
+def _get_args_and_env(esdata, esdata_override, kwargs):
+    env = os.environ.copy()
     args = [
-        os.path.join(prefix, 'elasticsearch'),
-        '-Enetwork.host=%s' % host,
-        '-Ehttp.port=%d' % port,
-        '-Epath.data=%s' % os.path.join(datadir, 'data'),
-        '-Epath.logs=%s' % os.path.join(datadir, 'logs'),
+        os.path.join(kwargs.get('prefix', ''), 'elasticsearch'),
     ]
-    if os.environ.get('TRAVIS'):
-        print('IN TRAVIS')
-        echo = True
-        args.append('-Epath.conf=%s/conf' % os.environ['TRAVIS_BUILD_DIR'])
-    elif os.path.exists('/etc/elasticsearch'):
-        print('NOT IN TRAVIS')
+    if esdata_override:
+        # ES configuration dir is specified, not default location.
+        # 'esdata' is ignored and kwargs values are overidden by elasticsearch.yml
+        esconfig = f"{esdata_override}/config"
+        if kwargs['version'] <= 5:
+            # How elasticsearch 5 sets config path
+            args.append(f"-Epath.conf={esconfig}")
+        else:
+            # How elasticsearch 6+ sets config path
+            env["ES_PATH_CONF"] = esconfig
+        jvm_options = _get_jvm_options(f"{esconfig}/jvm.options")
+        if jvm_options:
+            env['ES_JAVA_OPTS'] = jvm_options
+        return args, env
+    # Default 'esdata' location
+    args.extend([
+        '-Enetwork.host=%s' % kwargs.get('host', '127.0.0.1'),
+        '-Ehttp.port=%d' % kwargs.get('port', 9201),
+        '-Epath.data=%s' % os.path.join(esdata, 'data'),
+        '-Epath.logs=%s' % os.path.join(esdata, 'logs'),
+    ])
+    if _CIRCLE_BUILD and int(_CIRCLE_ES_VERSION) == 5:
         args.append('-Epath.conf=./conf')
-    print(args)
+    return args, env
+
+
+def _get_config_override(esdata, kwargs):
+    if _ENCD_CONFIG_DIR:
+        # If found, elasticsearch.yml config overrides vars like logs/data/host/port/etc...
+        es_data_dir = f"{_ENCD_CONFIG_DIR}/elasticsearch/es{kwargs['version']}"
+        if kwargs.get('local_test', False):
+            es_data_dir += 'test'
+        if os.path.exists(f"{es_data_dir}/config/elasticsearch.yml"):
+            return es_data_dir
+    return None
+
+
+def _get_jvm_options(jvm_options_path):
+    with open(jvm_options_path, 'r') as fileh:
+        return ' '.join([
+            line.strip()
+            for line in fileh.readlines()
+            if line[0] == '-'
+        ])
+
+
+def _clear(esdata, esdata_override):
+    rm_dirs = [esdata]
+    if esdata_override:
+        esconfig = f"{esdata_override}/config"
+        rm_dirs.extend([f"{esdata_override}/data", f"{esdata_override}/logs", f"{esconfig}/scripts"])
+        es_keystore_path = f"{esconfig}/elasticsearch.keystore"
+        if os.path.exists(es_keystore_path):
+            os.remove(es_keystore_path)
+    for dir_path in rm_dirs:
+        if os.path.exists(dir_path):
+            shutil.rmtree(dir_path)
+
+
+def server_process(datadir, **kwargs):
+    kwargs['version'] = kwargs.get('version', _OSX_ES_VERSION)
+    esdata = os.path.join(datadir, 'data')
+    esdata_override = _get_config_override(esdata, kwargs)
+    if kwargs.get('clear', False):
+        _clear(esdata, esdata_override)
+    args, env = _get_args_and_env(esdata, esdata_override, kwargs)
     process = subprocess.Popen(
         args,
         close_fds=True,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
+        env=env,
     )
 
     SUCCESS_LINE = b'started\n'
-
     lines = []
     for line in iter(process.stdout.readline, b''):
-        if echo:
+        if kwargs.get('echo', False):
             sys.stdout.write(line.decode('utf-8'))
         lines.append(line)
         if line.endswith(SUCCESS_LINE):
@@ -45,7 +110,7 @@ def server_process(datadir, host='127.0.0.1', port=9201, prefix='', echo=False):
         msg = ('Process return code: %d\n' % code) + b''.join(lines).decode('utf-8')
         raise Exception(msg)
 
-    if not echo:
+    if not kwargs.get('echo', False):
         process.stdout.close()
     print('returning process')
     return process

src/snovault/tests/serverfixtures.py

@@ -74,7 +74,7 @@ def elasticsearch_server(request, elasticsearch_host_port):
     host, port = elasticsearch_host_port
     tmpdir = request.config._tmpdirhandler.mktemp('elasticsearch', numbered=True)
     tmpdir = str(tmpdir)
-    process = server_process(str(tmpdir), host=host, port=9201, echo=False)
+    process = server_process(str(tmpdir), local_test=True, host=host, port=9201, echo=False)
     print('PORT CHANGED')
     yield 'http://%s:%d' % (host, 9201)