Release.1.2

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Add devices x,y,z , configure settings a,b,c
+2. Do API-call to URL x with data y
+3. See error in job output / syslog output 
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Outputs**
+If applicable, add JSON outputs and logs. Also include relevant parts of git settings or templates if needed.
+
+**Environment:**
+ - Setup [eg docker or standalone]
+ - Version [e.g. v1.1]
+ - Client [e.g. curl, webui]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.gitignore

@@ -25,6 +25,7 @@ downloads/
 eggs/
 .eggs/
 lib/
+lib64
 lib64/
 parts/
 sdist/
@@ -36,6 +37,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+bin/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -59,6 +61,7 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+docker/coverage/
 
 # Translations
 *.mo
@@ -106,6 +109,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+pyvenv.cfg
 
 x# Spyder project settings
 .spyderproject

alembic/versions/9d01bce3c835_add_job_status_aborting_and_start_.py

@@ -0,0 +1,30 @@
+"""add job status ABORTING and start_arguments field to job to save starting arguments in history
+
+Revision ID: 9d01bce3c835
+Revises: 8a635012afa7
+Create Date: 2020-11-02 10:03:03.293297
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '9d01bce3c835'
+down_revision = '8a635012afa7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('job', sa.Column('start_arguments', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    op.execute("COMMIT")
+    op.execute("ALTER TYPE jobstatus ADD VALUE 'ABORTING' AFTER 'ABORTED'")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('job', 'start_arguments')
+    # ### end Alembic commands ###

docker/api/Dockerfile

@@ -2,50 +2,54 @@ FROM debian:buster
 ARG BUILDBRANCH=develop
 
 # Create directories
-RUN mkdir -p /opt/cnaas
-RUN mkdir /opt/cnaas/templates
-RUN mkdir /opt/cnaas/settings
-RUN mkdir /etc/cnaas-nms
+RUN mkdir -p /opt/cnaas/templates/
+RUN mkdir /opt/cnaas/settings/
+RUN mkdir /etc/cnaas-nms/
 
 # Copy configuration files
-COPY config/db_config.yml /etc/cnaas-nms/db_config.yml
-COPY config/api.yml /etc/cnaas-nms/api.yml
-COPY config/repository.yml /etc/cnaas-nms/repository.yml
-COPY config/plugins.yml /etc/cnaas-nms/plugins.yml
+# db_config.yml, api.yml, repository.yml, plugins.yml
+COPY config/*.yml /etc/cnaas-nms/
 
 # Setup script
-COPY cnaas-setup.sh /opt/cnaas/cnaas-setup.sh
-RUN /opt/cnaas/cnaas-setup.sh $BUILDBRANCH
+COPY cnaas-setup.sh /opt/cnaas/
+COPY cnaas-setup-branch.sh /opt/cnaas/
+RUN /opt/cnaas/cnaas-setup.sh
 
 # Prepare for supervisord, uwsgi, ngninx
 COPY nosetests.sh /opt/cnaas/
 COPY exec-pre-app.sh /opt/cnaas/
-COPY config/uwsgi.ini /opt/cnaas/venv/cnaas-nms/
+COPY createca.sh /opt/cnaas/
+COPY --chown=root:www-data config/uwsgi.ini /opt/cnaas/venv/cnaas-nms/
 COPY config/supervisord_app.conf /etc/supervisor/supervisord.conf
 COPY config/nginx_app.conf /etc/nginx/sites-available/
 COPY config/nginx.conf /etc/nginx/
 COPY cert/* /etc/nginx/conf.d/
 
-# Websocket test client
-RUN mkdir /opt/cnaas/static
-COPY client.html /opt/cnaas/static
-
 # Give nginx some special treatment
 RUN unlink /etc/nginx/sites-enabled/default
 RUN ln -s /etc/nginx/sites-available/nginx_app.conf /etc/nginx/sites-enabled/default
-RUN chown www-data:www-data /var/log/nginx
-RUN chown www-data:www-data /etc/cnaas-nms/repository.yml
-RUN chown -R  www-data:www-data /var/log/nginx/
-RUN chown -R www-data:www-data /var/lib/nginx
-RUN chown www-data:www-data /var/lib/nginx/
+RUN chown www-data:www-data /etc/cnaas-nms/*.yml
+RUN chown -R www-data:www-data /var/log/nginx/
+RUN chown -R www-data:www-data /var/lib/nginx/
+RUN chown -R root:www-data /etc/nginx/ && \
+    chmod -R u=rwX,g=rX,o= /etc/nginx/
 # Give permission for API to clone/sync repos
-RUN chown www-data:www-data /opt/cnaas
-RUN chown -R www-data:www-data /opt/cnaas/templates
-RUN chown -R www-data:www-data /opt/cnaas/settings
+RUN chown -R root:www-data /opt/cnaas/ && \
+    chmod -R u=rwX,g=rX,o= /opt/cnaas/
+RUN chown -R www-data:www-data /opt/cnaas/templates/
+RUN chown -R www-data:www-data /opt/cnaas/settings/
+# Give permission for devicecert store
+RUN mkdir /tmp/devicecerts
+RUN chown -R www-data:www-data /tmp/devicecerts && \
+    chmod -R u=rwX,g=,o= /tmp/devicecerts
 # Give permission for unittests
-RUN chown root:www-data /opt/cnaas/nosetests.sh
-RUN chmod g+rx /opt/cnaas/nosetests.sh
-RUN chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src
+RUN chown root:www-data /opt/cnaas/*.sh && \
+    chmod g+rx /opt/cnaas/*.sh
+RUN chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/
+
+# Branch specific, don't cache
+ADD "https://api.github.com/repos/SUNET/cnaas-nms/git/refs/heads/" latest_commit
+RUN /opt/cnaas/cnaas-setup-branch.sh $BUILDBRANCH
 
 
 # Expose HTTPS

docker/api/cnaas-setup-branch.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Temporary for testing new branch
+if [ "$1" != "develop" ] ; then
+	cd /opt/cnaas/venv/cnaas-nms/
+	git checkout --track origin/$1
+	python3 -m pip install -r requirements.txt
+fi

docker/api/cnaas-setup.sh

@@ -45,15 +45,6 @@ cd cnaas-nms/
 git checkout master
 python3 -m pip install -r requirements.txt
 
-# Temporary for testing new branch
-if [ "$1" != "develop" ] ; then
-	cd /opt/cnaas/venv/cnaas-nms/
-	git remote update
-	git fetch
-	git checkout --track origin/$1
-	python3 -m pip install -r requirements.txt
-fi
-
 chown -R www-data:www-data /opt/cnaas/settings
 chown -R www-data:www-data /opt/cnaas/templates
 #rm -rf /var/lib/apt/lists/*

docker/api/config/api.yml

@@ -2,3 +2,7 @@ host: 0.0.0.0
 httpd_url: "https://cnaas_httpd:1443/api/v1.0/firmware"
 verify_tls: False
 jwtcert: /opt/cnaas/jwtcert/public.pem
+verify_tls_device: False
+cafile: /opt/cnaas/cacert/rootCA.crt
+cakeyfile: /opt/cnaas/cacert/rootCA.key
+certpath: /tmp/devicecerts/

docker/api/config/supervisord_app.conf

@@ -6,6 +6,15 @@ loglevel=debug
 pidfile=/tmp/supervisord.pid
 childlogdir=/tmp
 
+[inet_http_server]
+port=127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=http://127.0.0.1:9001
+
 [program:uwsgi]
 command = /usr/local/bin/uwsgi --ini /opt/cnaas/venv/cnaas-nms/uwsgi.ini
 autorestart=true

docker/api/createca.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+if [ -f /opt/cnaas/cacert/rootCA.key ] || [ -f /opt/cnaas/cacert/rootCA.crt ]
+then
+	exit 0
+fi
+
+cd /opt/cnaas/cacert
+umask 077
+openssl genrsa -out rootCA.key 4096
+openssl req -subj /C=/ST=/L=/O=/CN=cnaasNMSrootCA -x509 -new -nodes -key rootCA.key -sha256 -out rootCA.crt -days 7300
+chown root:www-data rootCA.*
+chmod 640 rootCA.*
+

docker/api/nosetests.sh

@@ -5,6 +5,15 @@ source venv/bin/activate
 
 cd venv/cnaas-nms/src
 
+export USERNAME_DHCP_BOOT="admin"
+export PASSWORD_DHCP_BOOT="abc123abc123"
+export USERNAME_DISCOVERED="admin"
+export PASSWORD_DISCOVERED="abc123abc123"
+export USERNAME_INIT="admin"
+export PASSWORD_INIT="abc123abc123"
+export USERNAME_MANAGED="admin"
+export PASSWORD_MANAGED="abc123abc123"
+
 nosetests --collect-only --with-id -v
 nosetests --with-coverage --cover-package=cnaas_nms -v
 cp .coverage /coverage/.coverage-nosetests

docker/dhcpd/cnaas-setup.sh

@@ -36,5 +36,5 @@ cd cnaas-nms/
 git checkout develop
 #python3 -m pip install -r requirements.txt
 #minimal requirements for just dhcp-hook:
-python3 -m pip install requests pyyaml netaddr
+python3 -m pip install requests pyyaml netaddr jinja2
 

docker/dhcpd/dhcpd.sh

@@ -19,6 +19,14 @@ then
 	fi
 fi
 
+if [ -e "/opt/cnaas/etc/dhcpd/gen-dhcpd.py" ] && \
+       [ -e "/opt/cnaas/etc/dhcpd/dhcpd.j2" ] && \
+       [ -e "/opt/cnaas/etc/dhcpd/dhcpd.yaml" ]; then
+    source /opt/cnaas/venv/bin/activate
+    cd /opt/cnaas/etc/dhcpd/
+    python3 gen-dhcpd.py > /opt/cnaas/dhcpd.conf
+fi
+
 #cd /opt/cnaas/venv/cnaas-nms
 #git pull
 

docker/docker-compose.yaml

@@ -40,6 +40,9 @@ services:
       - type: volume
         source: cnaas-jwtcert
         target: /opt/cnaas/jwtcert
+      - type: volume
+        source: cnaas-cacert
+        target: /opt/cnaas/cacert
 
   cnaas_httpd:
     image:
@@ -78,7 +81,7 @@ services:
       - 5432:5432
     environment:
       - POSTGRES_USER=cnaas
-      - POSTGRES_PASSWD=cnaas
+      - POSTGRES_PASSWORD=cnaas
       - POSTGRES_DB=cnaas
     ports:
       - 5432:5432
@@ -118,3 +121,5 @@ volumes:
     external: true
   cnaas-jwtcert:
     external: true
+  cnaas-cacert:
+    external: true

docker/jwt-cert/public.pem

@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPW8bkkVIq4BX8eWwlUOUYbJhiGDv
+K/6xY5T0BsvV6pbMoIUfgeThVOq5I3CmXxLt+qyPska6ol9fTN7woZLsCg==
+-----END PUBLIC KEY-----

docker/postgres/Dockerfile

@@ -1,5 +1,6 @@
 FROM postgres:11
 
 COPY nms.sql /docker-entrypoint-initdb.d/
+RUN chown postgres:postgres /docker-entrypoint-initdb.d/*.sql
 
 EXPOSE 5432