actian (#504)

* update dependencies

* update dependencies

* update dependencies

* actian integration

* fix certifi

* actian pipeline

.gitignore

@@ -12,3 +12,4 @@ version.py
 sdc-data*/
 containers/streamsets/python-libs/
 agent/backup-data/
+/agent/actian_config/actian-client/

Makefile

@@ -135,6 +135,9 @@ test-mssql: bootstrap
 test-databricks: bootstrap
 	$(DOCKER_TEST) tests/test_input/test_1/test_databricks.py
 
+test-actian: bootstrap
+	$(DOCKER_TEST) tests/test_input/test_2/test_actian.py
+
 test-snmp: bootstrap run-snmpsim
 	$(DOCKER_TEST) tests/test_input/test_2/test_snmp.py
 	$(DOCKER_TEST) tests/test_pipelines/test_2/test_snmp.py

agent/Dockerfile.actian.local

@@ -0,0 +1,34 @@
+FROM tiangolo/uwsgi-nginx-flask:python3.10
+
+WORKDIR /usr/src/app
+
+COPY . .
+ADD .bashrc /root/.bashrc
+COPY ./custom_entrypoint.sh /
+COPY supervisord.conf /etc/supervisor/supervisord.conf
+COPY nginx.conf /etc/nginx/nginx.conf
+COPY uwsgi-nginx-entrypoint.sh /uwsgi-nginx-entrypoint.sh
+COPY uwsgi.ini /etc/uwsgi/uwsgi.ini
+RUN chmod +x /custom_entrypoint.sh /uwsgi-nginx-entrypoint.sh
+
+ENV STREAMSETS_USERNAME="admin" STREAMSETS_PASSWORD="admin" STREAMSETS_URL="http://localhost:18630" \
+    LOG_FILE_PATH="/var/log/agent/agent.log" ANODOT_API_URL="https://api.anodot.com" ENV_PROD='true' \
+    GIT_SHA1=local-build VALIDATION_ENABLED="true" FLASK_APP="/usr/src/app/src/agent/api/main.py" FLASK_DEBUG=1 \
+    prometheus_multiproc_dir="/opt/metrics"
+
+# actian dependencies
+RUN apt-get update && apt-get install sudo
+COPY ./actian_config/actian-client /tmp/actian-client
+RUN /tmp/actian-client/client_install.sh -express
+COPY ./actian_config/odbcinst.ini /etc/odbcinst.ini
+ENV II_ODBC_WCHAR_SIZE=2 II_SYSTEM="/opt/Actian/Actian_Client" \
+    LD_LIBRARY_PATH=/lib:/usr/lib:/opt/Actian/Actian_Client/ingres/lib:/opt/Actian/Actian_Client/ingres/lib/lp32
+
+RUN ./install_requirements.sh
+RUN python setup.py develop
+
+RUN groupadd -g 10001 agent && useradd -r -u 10001 -g agent agent
+RUN mkdir /var/log/agent /opt/metrics
+
+ENTRYPOINT ["/custom_entrypoint.sh"]
+CMD flask run --host=0.0.0.0 --port=8080

agent/Dockerfile.local

@@ -23,4 +23,4 @@ RUN groupadd -g 10001 agent && useradd -r -u 10001 -g agent agent
 RUN mkdir /var/log/agent /opt/metrics
 
 ENTRYPOINT ["/custom_entrypoint.sh"]
-CMD flask run --host=0.0.0.0 --port=80
+CMD flask run --host=0.0.0.0 --port=8080

agent/actian_config/docker-compose-dev.yml

@@ -0,0 +1,57 @@
+services:
+  dc:
+    platform: linux/amd64
+    build:
+      context: ../../containers/streamsets
+    ports:
+      - "18630:18630"
+    restart: on-failure
+    volumes:
+      - ./sdc-data:/data
+      - ./sdc-conf:/etc/sdc_conf
+
+  agent:
+    platform: linux/amd64
+    build:
+      context: ..
+      dockerfile: Dockerfile.actian.local
+    restart: always
+    environment:
+      STREAMSETS_USERNAME: 'admin'
+      STREAMSETS_PASSWORD: 'admin'
+      STREAMSETS_URL: 'http://dc:18630'
+      ENV_PROD: 'false'
+      AGENT_DB_HOST: 'db'
+      AGENT_DB_USER: 'agent'
+      AGENT_DB_PASSWORD: 'agent'
+      GIT_SHA1: 'local'
+      AGENT_URL: 'http://anodot-agent'
+      PGPASSWORD: 'agent'
+      TLS_VERSION: '1.1'
+      LOG_LEVEL: 'DEBUG'
+    depends_on:
+      - dc
+      - db
+    volumes:
+      - ../backup-data:/usr/src/app/backup-data
+      - ../output:/output
+      - ../:/usr/src/app
+    ports:
+      - "80:80"
+
+  db:
+    image: postgres:15
+    restart: always
+    environment:
+      POSTGRES_USER: agent
+      POSTGRES_PASSWORD: agent
+      POSTGRES_DB: agent
+    ports:
+      - "5432:5432"
+    volumes:
+      - db-data:/var/lib/postgresql/data
+
+
+volumes:
+  agent-data:
+  db-data:

agent/actian_config/odbcinst.ini

@@ -0,0 +1,9 @@
+[ODBC]
+Trace = yes
+TraceFile = /tmp/odbctrace.log
+
+[ODBC Drivers]
+Ingres=Installed
+
+[Ingres]
+Driver=/opt/Actian/Actian_Client/ingres/lib/libiiodbcdriver.1.so

agent/install_requirements.sh

@@ -7,6 +7,6 @@ wget --no-check-certificate --quiet -O - https://www.postgresql.org/media/keys/A
 # using stretch instead of `lsb_release -cs`-pgdg main as it's not working with the current version
 echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 
-apt update && apt install -y --reinstall vim netcat curl nano ca-certificates postgresql-client-15 librrd-dev default-mysql-client kafkacat
+apt update && apt install -y --reinstall vim netcat curl nano ca-certificates postgresql-client-15 librrd-dev default-mysql-client kafkacat unixodbc-dev
 
 pip install --upgrade pip && pip install -r requirements.txt

agent/requirements.txt

@@ -57,4 +57,5 @@ pysnmp-mibs==0.1.6
 psycopg2==2.9.1
 mysqlclient==1.4.6
 WTForms==3.1.2
-WTForms-SQLAlchemy==0.4.1
+WTForms-SQLAlchemy==0.4.1
+pyodbc==5.1.0

agent/src/agent/api/main.py

@@ -20,6 +20,7 @@
 app.register_blueprint(pipeline.pipelines)
 app.register_blueprint(scripts.scripts)
 app.register_blueprint(monitoring_bp)
+app.register_blueprint(data_extractors.actian)
 app.register_blueprint(data_extractors.cacti)
 app.register_blueprint(data_extractors.observium)
 app.register_blueprint(data_extractors.rrd)

agent/src/agent/api/routes/data_extractors/__init__.py

@@ -1,3 +1,4 @@
+from .actian import actian
 from .cacti import cacti
 from .observium import observium
 from .snmp import snmp

agent/src/agent/api/routes/data_extractors/actian.py

@@ -0,0 +1,16 @@
+from flask import jsonify, Blueprint, request
+from agent import pipeline, data_extractor
+from agent.api.routes import needs_pipeline
+
+actian = Blueprint('actian_source', __name__)
+
+
+@actian.route('/data_extractors/actian/<pipeline_id>', methods=['GET'])
+@needs_pipeline
+def read(pipeline_id: str):
+    pipeline_ = pipeline.repository.get_by_id(pipeline_id)
+    offset = request.args.get('offset')
+    if not offset:
+        return jsonify('No offset provided'), 400
+    metrics = data_extractor.actian.extract_metrics(pipeline_, int(offset))
+    return jsonify(metrics)

agent/src/agent/data_extractor/__init__.py

@@ -3,3 +3,4 @@
 from . import snmp
 from . import topology
 from . import rrd
+from . import actian

agent/src/agent/data_extractor/actian/__init__.py

@@ -0,0 +1 @@
+from .actian import extract_metrics

agent/src/agent/data_extractor/actian/actian.py

@@ -0,0 +1,21 @@
+import pyodbc
+from agent import source
+from agent.pipeline import Pipeline
+from agent.modules import logger
+
+logger_ = logger.get_logger(__name__)
+
+
+def extract_metrics(pipeline_: Pipeline, offset: int) -> list:
+    cnx = pyodbc.connect(pipeline_.source.config[source.ActianSource.CONNECTION_STRING])
+    cursor = cnx.cursor()
+    timestamp_to_unix = f'UNIX_TIMESTAMP({pipeline_.timestamp_path})'
+    timestamp_condition = f'{timestamp_to_unix} >= {offset} AND {timestamp_to_unix} < {offset} + {pipeline_.interval}'
+    query = pipeline_.query.replace('{TIMESTAMP_CONDITION}', timestamp_condition)
+    logger_.info(f'Executing query: {query}')
+    cursor.execute(query)
+    rows = cursor.fetchall()
+    column_names = [c[0] for c in cursor.description]
+    return [{c: row.__getattribute__(c) for c in column_names} for row in rows]
+
+

agent/src/agent/modules/constants.py

@@ -14,7 +14,7 @@
 VALIDATION_ENABLED = os.environ.get('VALIDATION_ENABLED', 'true') == 'true'
 DISABLE_PIPELINE_ERROR_NOTIFICATIONS = os.environ.get('DISABLE_PIPELINE_ERROR_NOTIFICATIONS', 'false') == 'true'
 
-AGENT_DEFAULT_URL = os.environ.get('AGENT_URL', 'http://anodot-agent')
+AGENT_DEFAULT_URL = os.environ.get('AGENT_URL', 'http://agent:8080')
 
 ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
 

agent/src/agent/modules/validator.py

@@ -1,6 +1,7 @@
 import os
 import sqlalchemy
 import subprocess
+import pyodbc
 
 from urllib.parse import urlparse
 from agent.modules.tools import if_validation_enabled
@@ -36,6 +37,10 @@ def validate_mysql_connection(connection_string: str):
     eng.connect()
 
 
+def validate_actian_connection(connection_string: str):
+    pyodbc.connect(connection_string)
+
+
 def validate_python_file(file: str):
     if not os.path.isfile(file):
         raise ValidationException(f'No such file `{file}`')