diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..1feddd3f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Add devices x,y,z , configure settings a,b,c +2. Do API-call to URL x with data y +3. See error in job output / syslog output + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Outputs** +If applicable, add JSON outputs and logs. Also include relevant parts of git settings or templates if needed. + +**Environment:** + - Setup [eg docker or standalone] + - Version [e.g. v1.1] + - Client [e.g. curl, webui] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..bbcbbe7d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.gitignore b/.gitignore index 480c3599..da7d37f4 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ downloads/ eggs/ .eggs/ lib/ +lib64 lib64/ parts/ sdist/ @@ -36,6 +37,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +bin/ # PyInstaller # Usually these files are written by a python script from a template @@ -59,6 +61,7 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ +docker/coverage/ # Translations *.mo @@ -106,6 +109,7 @@ venv/ ENV/ env.bak/ venv.bak/ +pyvenv.cfg x# Spyder project settings .spyderproject diff --git a/alembic/versions/9d01bce3c835_add_job_status_aborting_and_start_.py b/alembic/versions/9d01bce3c835_add_job_status_aborting_and_start_.py new file mode 100644 index 00000000..a8facec7 --- /dev/null +++ b/alembic/versions/9d01bce3c835_add_job_status_aborting_and_start_.py @@ -0,0 +1,30 @@ +"""add job status ABORTING and start_arguments field to job to save starting arguments in history + +Revision ID: 9d01bce3c835 +Revises: 8a635012afa7 +Create Date: 2020-11-02 10:03:03.293297 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '9d01bce3c835' +down_revision = '8a635012afa7' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('job', sa.Column('start_arguments', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.execute("COMMIT") + op.execute("ALTER TYPE jobstatus ADD VALUE 'ABORTING' AFTER 'ABORTED'") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('job', 'start_arguments') + # ### end Alembic commands ### diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index 86505e08..edc2c983 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -2,50 +2,54 @@ FROM debian:buster ARG BUILDBRANCH=develop # Create directories -RUN mkdir -p /opt/cnaas -RUN mkdir /opt/cnaas/templates -RUN mkdir /opt/cnaas/settings -RUN mkdir /etc/cnaas-nms +RUN mkdir -p /opt/cnaas/templates/ +RUN mkdir /opt/cnaas/settings/ +RUN mkdir /etc/cnaas-nms/ # Copy configuration files -COPY config/db_config.yml /etc/cnaas-nms/db_config.yml -COPY config/api.yml /etc/cnaas-nms/api.yml -COPY config/repository.yml /etc/cnaas-nms/repository.yml -COPY config/plugins.yml /etc/cnaas-nms/plugins.yml +# db_config.yml, api.yml, repository.yml, plugins.yml +COPY config/*.yml /etc/cnaas-nms/ # Setup script -COPY cnaas-setup.sh /opt/cnaas/cnaas-setup.sh -RUN /opt/cnaas/cnaas-setup.sh $BUILDBRANCH +COPY cnaas-setup.sh /opt/cnaas/ +COPY cnaas-setup-branch.sh /opt/cnaas/ +RUN /opt/cnaas/cnaas-setup.sh # Prepare for supervisord, uwsgi, ngninx COPY nosetests.sh /opt/cnaas/ COPY exec-pre-app.sh /opt/cnaas/ -COPY config/uwsgi.ini /opt/cnaas/venv/cnaas-nms/ +COPY createca.sh /opt/cnaas/ +COPY --chown=root:www-data config/uwsgi.ini /opt/cnaas/venv/cnaas-nms/ COPY config/supervisord_app.conf /etc/supervisor/supervisord.conf COPY config/nginx_app.conf /etc/nginx/sites-available/ COPY config/nginx.conf /etc/nginx/ COPY cert/* /etc/nginx/conf.d/ -# Websocket test client -RUN mkdir /opt/cnaas/static -COPY client.html /opt/cnaas/static - # Give nginx some special treatment RUN unlink /etc/nginx/sites-enabled/default RUN ln -s /etc/nginx/sites-available/nginx_app.conf /etc/nginx/sites-enabled/default -RUN chown www-data:www-data /var/log/nginx -RUN chown www-data:www-data /etc/cnaas-nms/repository.yml -RUN chown -R www-data:www-data /var/log/nginx/ -RUN chown -R www-data:www-data /var/lib/nginx -RUN chown www-data:www-data /var/lib/nginx/ +RUN chown www-data:www-data /etc/cnaas-nms/*.yml +RUN chown -R www-data:www-data /var/log/nginx/ +RUN chown -R www-data:www-data /var/lib/nginx/ +RUN chown -R root:www-data /etc/nginx/ && \ + chmod -R u=rwX,g=rX,o= /etc/nginx/ # Give permission for API to clone/sync repos -RUN chown www-data:www-data /opt/cnaas -RUN chown -R www-data:www-data /opt/cnaas/templates -RUN chown -R www-data:www-data /opt/cnaas/settings +RUN chown -R root:www-data /opt/cnaas/ && \ + chmod -R u=rwX,g=rX,o= /opt/cnaas/ +RUN chown -R www-data:www-data /opt/cnaas/templates/ +RUN chown -R www-data:www-data /opt/cnaas/settings/ +# Give permission for devicecert store +RUN mkdir /tmp/devicecerts +RUN chown -R www-data:www-data /tmp/devicecerts && \ + chmod -R u=rwX,g=,o= /tmp/devicecerts # Give permission for unittests -RUN chown root:www-data /opt/cnaas/nosetests.sh -RUN chmod g+rx /opt/cnaas/nosetests.sh -RUN chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src +RUN chown root:www-data /opt/cnaas/*.sh && \ + chmod g+rx /opt/cnaas/*.sh +RUN chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/ + +# Branch specific, don't cache +ADD "https://api.github.com/repos/SUNET/cnaas-nms/git/refs/heads/" latest_commit +RUN /opt/cnaas/cnaas-setup-branch.sh $BUILDBRANCH # Expose HTTPS diff --git a/docker/api/client.html b/docker/api/client.html deleted file mode 100644 index ffa4aa99..00000000 --- a/docker/api/client.html +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - diff --git a/docker/api/cnaas-setup-branch.sh b/docker/api/cnaas-setup-branch.sh new file mode 100755 index 00000000..b2d73100 --- /dev/null +++ b/docker/api/cnaas-setup-branch.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e +set -x + +# Temporary for testing new branch +if [ "$1" != "develop" ] ; then + cd /opt/cnaas/venv/cnaas-nms/ + git checkout --track origin/$1 + python3 -m pip install -r requirements.txt +fi diff --git a/docker/api/cnaas-setup.sh b/docker/api/cnaas-setup.sh index c779fc01..2332ade3 100755 --- a/docker/api/cnaas-setup.sh +++ b/docker/api/cnaas-setup.sh @@ -45,15 +45,6 @@ cd cnaas-nms/ git checkout master python3 -m pip install -r requirements.txt -# Temporary for testing new branch -if [ "$1" != "develop" ] ; then - cd /opt/cnaas/venv/cnaas-nms/ - git remote update - git fetch - git checkout --track origin/$1 - python3 -m pip install -r requirements.txt -fi - chown -R www-data:www-data /opt/cnaas/settings chown -R www-data:www-data /opt/cnaas/templates #rm -rf /var/lib/apt/lists/* diff --git a/docker/api/config/api.yml b/docker/api/config/api.yml index f21953c3..7bc6dbcf 100644 --- a/docker/api/config/api.yml +++ b/docker/api/config/api.yml @@ -2,3 +2,7 @@ host: 0.0.0.0 httpd_url: "https://cnaas_httpd:1443/api/v1.0/firmware" verify_tls: False jwtcert: /opt/cnaas/jwtcert/public.pem +verify_tls_device: False +cafile: /opt/cnaas/cacert/rootCA.crt +cakeyfile: /opt/cnaas/cacert/rootCA.key +certpath: /tmp/devicecerts/ diff --git a/docker/api/config/supervisord_app.conf b/docker/api/config/supervisord_app.conf index 2805fb8b..2b21dbf4 100644 --- a/docker/api/config/supervisord_app.conf +++ b/docker/api/config/supervisord_app.conf @@ -6,6 +6,15 @@ loglevel=debug pidfile=/tmp/supervisord.pid childlogdir=/tmp +[inet_http_server] +port=127.0.0.1:9001 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=http://127.0.0.1:9001 + [program:uwsgi] command = /usr/local/bin/uwsgi --ini /opt/cnaas/venv/cnaas-nms/uwsgi.ini autorestart=true diff --git a/docker/api/createca.sh b/docker/api/createca.sh new file mode 100755 index 00000000..f3bb462a --- /dev/null +++ b/docker/api/createca.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ -f /opt/cnaas/cacert/rootCA.key ] || [ -f /opt/cnaas/cacert/rootCA.crt ] +then + exit 0 +fi + +cd /opt/cnaas/cacert +umask 077 +openssl genrsa -out rootCA.key 4096 +openssl req -subj /C=/ST=/L=/O=/CN=cnaasNMSrootCA -x509 -new -nodes -key rootCA.key -sha256 -out rootCA.crt -days 7300 +chown root:www-data rootCA.* +chmod 640 rootCA.* + diff --git a/docker/api/nosetests.sh b/docker/api/nosetests.sh index a747025b..1d572ba0 100755 --- a/docker/api/nosetests.sh +++ b/docker/api/nosetests.sh @@ -5,6 +5,15 @@ source venv/bin/activate cd venv/cnaas-nms/src +export USERNAME_DHCP_BOOT="admin" +export PASSWORD_DHCP_BOOT="abc123abc123" +export USERNAME_DISCOVERED="admin" +export PASSWORD_DISCOVERED="abc123abc123" +export USERNAME_INIT="admin" +export PASSWORD_INIT="abc123abc123" +export USERNAME_MANAGED="admin" +export PASSWORD_MANAGED="abc123abc123" + nosetests --collect-only --with-id -v nosetests --with-coverage --cover-package=cnaas_nms -v cp .coverage /coverage/.coverage-nosetests diff --git a/docker/dhcpd/cnaas-setup.sh b/docker/dhcpd/cnaas-setup.sh index 77e8d664..6796aad9 100755 --- a/docker/dhcpd/cnaas-setup.sh +++ b/docker/dhcpd/cnaas-setup.sh @@ -36,5 +36,5 @@ cd cnaas-nms/ git checkout develop #python3 -m pip install -r requirements.txt #minimal requirements for just dhcp-hook: -python3 -m pip install requests pyyaml netaddr +python3 -m pip install requests pyyaml netaddr jinja2 diff --git a/docker/dhcpd/dhcpd.sh b/docker/dhcpd/dhcpd.sh index 04cfd379..5888ee83 100755 --- a/docker/dhcpd/dhcpd.sh +++ b/docker/dhcpd/dhcpd.sh @@ -19,6 +19,14 @@ then fi fi +if [ -e "/opt/cnaas/etc/dhcpd/gen-dhcpd.py" ] && \ + [ -e "/opt/cnaas/etc/dhcpd/dhcpd.j2" ] && \ + [ -e "/opt/cnaas/etc/dhcpd/dhcpd.yaml" ]; then + source /opt/cnaas/venv/bin/activate + cd /opt/cnaas/etc/dhcpd/ + python3 gen-dhcpd.py > /opt/cnaas/dhcpd.conf +fi + #cd /opt/cnaas/venv/cnaas-nms #git pull diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index dbc7aacc..5d5ff3cf 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -40,6 +40,9 @@ services: - type: volume source: cnaas-jwtcert target: /opt/cnaas/jwtcert + - type: volume + source: cnaas-cacert + target: /opt/cnaas/cacert cnaas_httpd: image: @@ -78,7 +81,7 @@ services: - 5432:5432 environment: - POSTGRES_USER=cnaas - - POSTGRES_PASSWD=cnaas + - POSTGRES_PASSWORD=cnaas - POSTGRES_DB=cnaas ports: - 5432:5432 @@ -118,3 +121,5 @@ volumes: external: true cnaas-jwtcert: external: true + cnaas-cacert: + external: true diff --git a/docker/jwt-cert/public.pem b/docker/jwt-cert/public.pem new file mode 100644 index 00000000..1ebfc733 --- /dev/null +++ b/docker/jwt-cert/public.pem @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPW8bkkVIq4BX8eWwlUOUYbJhiGDv +K/6xY5T0BsvV6pbMoIUfgeThVOq5I3CmXxLt+qyPska6ol9fTN7woZLsCg== +-----END PUBLIC KEY----- diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile index d968099d..f33b5201 100644 --- a/docker/postgres/Dockerfile +++ b/docker/postgres/Dockerfile @@ -1,5 +1,6 @@ FROM postgres:11 COPY nms.sql /docker-entrypoint-initdb.d/ +RUN chown postgres:postgres /docker-entrypoint-initdb.d/*.sql EXPOSE 5432 diff --git a/docs/apiref/devices.rst b/docs/apiref/devices.rst index 9afb39fd..fc602fb9 100644 --- a/docs/apiref/devices.rst +++ b/docs/apiref/devices.rst @@ -227,6 +227,69 @@ to allow apply config live run as well. This will schedule a job to send the configuration to the device. +Initialize check +---------------- + +Before initializing a new device you can run a pre-check API call. This will +perform some basic device state checks and check that compatible LLDP +neighbors are found. For access devices it will try and find a compatible +mgmtdomain and for core/dist devices it will check that interfaces facing +neighbors are set to the correct ifclass. It is possible that the init will +fail even if the initcheck passed. + +To test if a device is compatible for DIST ZTP run: + +:: + + curl https://localhost/api/v1.0/device_initcheck/45 -d '{"hostname": "dist3", "device_type": "DIST"}' -X POST -H "Content-Type: application/json" + +Example output: + +:: + + { + "status": "success", + "data": { + "linknets": [ + { + "description": null, + "device_a_hostname": "dist3", + "device_a_ip": "10.198.0.0", + "device_a_port": "Ethernet3", + "device_b_hostname": "core1", + "device_b_ip": "10.198.0.1", + "device_b_port": "Ethernet3", + "ipv4_network": "10.198.0.0/31", + "site_id": null + } + ], + "linknets_compatible": true, + "neighbors_compatible": false, + "neighbors_error": "Not enough linknets (1 of 2) were detected", + "parsed_args": { + "device_id": 2, + "new_hostname": "dist3", + "device_type": "DIST", + "neighbors": null + }, + "compatible": false + } + } + +Status success in this case means all checks were able to complete, but if +you check the "compatible" key it says false which means this device is +actually not compatible for DIST ZTP at the moment. We did find a compatible +linknet, but there were not enough neighboring devices of the correct device +type found. If you want to perform some non-standard configuration like trying +ZTP with just one neighbor you can manually specify what neighbors you expect +to see instead ("neighbors": ["core1"]). Other arguments that can be passed +to device_init should also be valid here, like "mlag_peer_id" and +"mlag_peer_hostname" for access MLAG pairs. + +If the checks can not be performed at all, like when the device is not found +or an invalid device type is specified the API call will return a 400 or 500 +error instead. + Initialize device ----------------- @@ -261,4 +324,40 @@ use this API call: This will schedule a job to log in to the device, get the facts and update the database. You can perform this action on both MANAGED and UNMANAGED devices. UNMANAGED devices might not be reachable so this could be a good test-call -before moving the device back to the MANAGED state. \ No newline at end of file +before moving the device back to the MANAGED state. + +Update interfaces +----------------- + +To update the list of available interfaces on an ACCESS device use this API call: + +:: + + curl https://localhost/api/v1.0/device_update_interfaces -d '{"hostname": "eosaccess"}' -X POST -H "Content-Type: application/json" + +This will schedule a job to log in to the device and get a list of physical +interfaces and put them in the interface database. Existing interfaces will +not be changed unless you specify "replace": true. Interfaces that no longer +exists on the device will be deleted from the interface database, +except for UPLINK and MLAG_PEER ports which will not be deleted automatically. +If you specify "delete_all": true then all interfaces will be removed, +including UPLINK and MLAG_PEER ports (dangerous!). If you want to re-populate +MLAG_PEER ports you have to specify the argument "mlag_peer_hostname" to +indicate what peer device you expect to see. + +Renew certificates +------------------ + +To manually request installation/renewal of a new device certificate use +the device_cert API: + +:: + + curl https://localhost/api/v1.0/device_cert -d '{"hostname": "eosdist1", "action": "RENEW"}' -X POST -H "Content-Type: application/json" + +This will schedule a job to generate a new key and certificate for the specified +device(s) and copy them to the device(s). The certificate will be signed by the +NMS CA (specified in api.yml). + +Either one of "hostname" or "group" arguments must be specified. The "action" +argument must be specified and the only valid action for now is "RENEW". diff --git a/docs/apiref/firmware.rst b/docs/apiref/firmware.rst index ee686c5e..55d7a7b6 100644 --- a/docs/apiref/firmware.rst +++ b/docs/apiref/firmware.rst @@ -152,7 +152,9 @@ The API method will accept a few parameters: * filename: Mandatory. Name of the new firmware, for example "test.swi". * url: Optional, can also be configured as an environment variable, FIRMQRE_URL. URL to the firmware storage, for example "http://hostname/firmware/". This should typically point to the CNaaS NMS server and files will be downloaded from the CNaaS HTTP server. * download: Optional, default is false. Only download the firmware. -* pre_flight: Optional, default is false. If false, check disk-space etc before downloading the firmware. +* pre_flight: Optional, default is false. If true, check disk-space etc before downloading the firmware. +* post_flight: Optional, default is false. If true, update OS version after the upgrade have been finished. +* post_waittime: Optional, default is 0. Defines the time we should wait before trying to connect to an updated device. * activate: Optional, default is false. Control whether we should install the new firmware or not. * reboot: Optional, default is false. When the firmware is downloaded, reboot the switch. * start_at: Schedule a firmware upgrade to be started sometime in the future. @@ -160,7 +162,7 @@ The API method will accept a few parameters: An example CURL command can look like this: :: - curl -k -s -H "Content-Type: application/json" -X POST https://hostname/api/v1.0/firmware/upgrade -d '{"group": "ACCESS", "filename": "test_firmware.swi", "url": "http://hostname/", "pre-flight": true, "download": true, "activate": true, "reboot": true, "start_at": "2019-12-24 00:00:00"}' + curl -k -s -H "Content-Type: application/json" -X POST https://hostname/api/v1.0/firmware/upgrade -d '{"group": "ACCESS", "filename": "test_firmware.swi", "url": "http://hostname/", "pre-flight": true, "download": true, "activate": true, "reboot": true, "start_at": "2019-12-24 00:00:00", "post_flight": true, "post_waittime": 600'} The output from the job will look like this: @@ -207,6 +209,12 @@ The output from the job will look like this: "result": "Device reboot done.", "diff": "", "failed": false + }, + { + "result": "Post-flight, OS version updated for device eosaccess, now 4.23.2F-15405360.4232F.", + "task_name": "arista_post_flight_check", + "diff": "", + "failed": false } ], "_totals": { diff --git a/docs/apiref/groups.rst b/docs/apiref/groups.rst index 4c133866..89e2f279 100644 --- a/docs/apiref/groups.rst +++ b/docs/apiref/groups.rst @@ -13,46 +13,70 @@ To show all groups the following REST call can be used: curl https://hostname/api/v1.0/groups -That will return a JSON structure with all the groups defined: +That will return a JSON structure with all group names +and the hostnames of all devices in the group: :: { "status": "success", "data": { - "status": "success", - "data": { - "groups": { - "group_0": [ - "testdevice_a", - ], - "group_1": [ - "testdevice_b", - "testdevice_c" - ] - } + "groups": { + "group_0": [ + "testdevice_a", + ], + "group_1": [ + "testdevice_b", + "testdevice_c" + ] } } } +Show specific group +------------------- -Define groups -------------- +To show a single group specify the group name in the path: + +:: + + curl https://hostname/api/v1.0/groups/mygroup + + +Show specific group OS versions +------------------------------- + +To show the OS versions of the devices in a group: + +:: -New groups can be defined in the settings template. + curl https://hostname/api/v1.0/groups/MY_EOS_DEVICES/os_versions -In the global settings, modify 'base_system.yml' and add the following -section: +Output: :: - - groups: - - group: - name: 'MY_NEW_GROUP' - regex: '.*' - - group: - name: 'ANOTHER_NEW_GROUP' - regex: '.*' - -As you can see, a name and a regex is expected. The regex will match -on the device hostnames and based on that add them to the group. + + { + "status": "success", + "data": { + "groups": { + "MY_EOS_DEVICES": { + "4.21.1.1F-10146868.42111F": [ + "eosaccess" + ], + "4.22.3M-14418192.4223M": [ + "eosdist1", + "eosdist2" + ] + } + } + } + } + + + +Define groups +------------- + +New groups can be defined in the settings repository. :ref:`settings_repo_ref` + diff --git a/docs/apiref/interfaces.rst b/docs/apiref/interfaces.rst index 099ac488..18bd3931 100644 --- a/docs/apiref/interfaces.rst +++ b/docs/apiref/interfaces.rst @@ -57,6 +57,8 @@ The configtype field must use some of these pre-defined values: - ACCESS_UNTAGGED: Use a static VLAN defined by name in the data field - ACCESS_TAGGED: Use a static list of VLANs defined by names in the data field - ACCESS_UPLINK: Uplink from access switch to dist switch +- ACCESS_DOWNLINK: Downlink from this access switch to another access switch +- MLAG_PEER: MLAG peer interface Update interface ---------------- @@ -156,6 +158,8 @@ To re-enable and unset description: curl https://hostname/api/v1.0/device/eosaccess/interfaces -d '{"interfaces": {"Ethernet1": {"data": {"enabled": true, "description": null}}}}' -X PUT -H "Content-Type: application/json" -H "Authorization: Bearer $JWT_AUTH_TOKEN" +If the list of interfaces does not match what currently exists on the device +you need to run the device_update_interfaces API call (see device API). Show interface states --------------------- diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst index a84b82d8..92e3f25e 100644 --- a/docs/changelog/index.rst +++ b/docs/changelog/index.rst @@ -1,6 +1,29 @@ Changelog ========= +Version 1.2.0 +------------- + +New features: + +- ZTP support for core and diste devices (#137) +- Init check API call to test if device is compatible for ZTP without commit (#136, #156) +- Option to have model-specific default interface settings (#135) +- Post-flight check for firmware upgrade (#139) +- Abort scheduled jobs, best-effort abort of running jobs (#142) +- API call to update existing interfaces on device after ZTP (#155) +- More settings for external BGP routing, DNS servers, internal VLANs (#143, #146, #152) +- Install NMS issued certificate on new devices during ZTP (#149) +- Switch to Nornir 3.0, improved whitespace rendering in templates (#148) + +Bug fixes: + +- Fix blocking websockets (#138) +- Fix access downlink port detection (#141) +- Post upgrade confighash mismatch (#145) +- Discover device duplicate jobs improvements (#151) +- Trim facts fields before saving in database (#153) + Version 1.1.0 ------------- diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst index 8cc122a6..3a88de25 100644 --- a/docs/configuration/index.rst +++ b/docs/configuration/index.rst @@ -23,6 +23,11 @@ Defines parameters for the API: - jwtcert: Defines the path to the public JWT certificate used to verify JWT tokens - httpd_url: URL to the httpd container containing firmware images - verify_tls: Verify certificate for connections to httpd/firmware server +- verify_tls_device: Verify TLS connections to devices, defaults to True +- cafile: Path to CA certificate used to verify device certificates. + If no path is specified then the system default CAs will be used. +- cakeyfile: Path to CA key, used to sign device certificates after generation. +- certpath: Path to store generated device certificates in. - allow_apply_config_liverun: Allow liverun on apply_config API call. Defaults to False. /etc/cnaas-nms/repository.yml diff --git a/docs/howto/index.rst b/docs/howto/index.rst index a0d30aa1..8d571ce7 100644 --- a/docs/howto/index.rst +++ b/docs/howto/index.rst @@ -112,6 +112,50 @@ container at the initial steps of the process, and also logs from the API container at later stages of the process. If the device gets stuck in the DHCP_BOOT process for example, it probably means the API can not log in to the device using the credentials and IP address saved in the database. The API -will retry connecting to the device 5 times with increasing delay between -each attempt. If you want to trigger more retries at a later point you can manually -call the discover_device API call and send the MAC and DHCP IP of the device. \ No newline at end of file +will retry connecting to the device 3 times with increasing delay between +each attempt. If you want to trigger more retries at a later point you can +manually call the discover_device API call and send the MAC and DHCP IP of the +device. New attempts to discover the device will also be made when the DHCP +lease is renewed or reaquired. + + +Zero-touch provisioning of fabric switch +---------------------------------------- + +You can also provision a new switch to be part of the (EVPN/VXLAN) fabric, that +is a switch with device_type DIST or CORE. Interfaces that connects between +CORE and DIST devices should be configured as ifclass "fabric" on both ends. +You can configure this in the settings repository via a device specific +setting or via a model specific setting (model setting might be preferable for +ZTP since you don't need to pre-provision new device hostnames in the settings +repository). + +To verify that interfaces are configured correctly and that LLDP neighbors +are seen you can use the device_initcheck API call (see devices API reference): + +:: + + curl https://localhost/api/v1.0/device_initcheck/45 -d '{"hostname": "dist3", "device_type": "DIST"}' -X POST -H "Content-Type: application/json" + +If all parameters are compatible you can start initialization: + +:: + + curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "dist3", "device_type": "DIST"}' -X POST -H "Content-Type: application/json" + +If LLDP neighbors are not seen or are not of the expected type (DIST type +expect neighbors of type CORE and vice versa) you can manually specify the +neighbors you want to verify connectivity to, but make sure you know what you +are doing and maybe set up console access if something goes wrong here: + +:: + + curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "dist3", "device_type": "DIST", "neighbors": ["dist1", "dist2"]}' -X POST -H "Content-Type: application/json" + +If you don't expect to see any LLDP neighbors at all and instead have +pre-configured some kind of uplink interfaces via ifclass custom interfaces +in settings for this device, you could also specify an empty list as +neighbors and in this case init will continue even if no LLDP neighbors were +detected. This is also very risky since you can't verify that interfaces +are connected correctly before sending configuration and possibly losing +connectivity to the device. diff --git a/docs/reporef/index.rst b/docs/reporef/index.rst index 5f4bf999..526c2901 100644 --- a/docs/reporef/index.rst +++ b/docs/reporef/index.rst @@ -64,7 +64,7 @@ Additional variables available for distribution switches: Populated from the links database table. - bgp_evpn_peers: A list of dictionaries with the keys: "peer_hostname", "peer_infra_lo", "peer_asn". - Contains one entry per hostname specified in settings->evpn_spines. Used to build + Contains one entry per hostname specified in settings->evpn_peers. Used to build eBGP peering for EVPN between loopbacks. - mgmtdomains: A list of dictionaries with the keys: "ipv4_gw", "vlan", "description", "esi_mac". @@ -75,6 +75,8 @@ Additional variables available for distribution switches: All settings configured in the settings repository are also exposed to the templates. +.. _settings_repo_ref: + settings -------- @@ -87,17 +89,19 @@ The directory structure looks like this: - global * groups.yml: Definition of custom device groups - * vxlans.yml: Definition of VXLAN/VLANs * routing.yml: Definition of global routing settings like fabric underlay and VRFs + * vxlans.yml: Definition of VXLAN/VLANs * base_system.yml: Base system settings - core * base_system.yml: Base system settings + * interfaces_.yml: Model specific default interface settings - dist * base_system.yml: Base system settings + * interfaces_.yml: Model specific default interface settings - access: @@ -111,6 +115,34 @@ The directory structure looks like this: + interfaces.yml + routing.yml +groups.yml: + +Contains a dictionary named "groups", that contains a list of groups. +Each group is defined as a dictionary with a single key named "group", +and that key contains a dictionary with two keys: + +- name: A string representing a name. No spaces. +- regex: A Python style regex that matches on device hostnames + +All devices that matches the regex will be included in the group. + +:: + + --- + groups: + - group: + name: 'ALL' + regex: '.*' + - group: + name: 'BORDER_DIST' + regex: '(south-dist0[1-2]|north-dist0[1-2])' + - group: + name: 'DIST_EVEN' + regex: '.*-dist[0-9][02468]' + - group: + name: 'DIST_ODD' + regex: '.*-dist[0-9][13579]' + routing.yml: Can contain the following dictionaries with specified keys: @@ -127,7 +159,8 @@ Can contain the following dictionaries with specified keys: * hostname: A hostname of a CORE (or DIST) device from the device database. The other DIST switches participating in the VXLAN/EVPN fabric will establish - eBGP connections to these devices. + eBGP connections to these devices. If an empty list is provided all CORE + devices will be added as evpn_peers instead. - vrfs: @@ -149,6 +182,12 @@ Can contain the following dictionaries with specified keys: * name: Name/description of route (optional, defaults to "undefined") * cli_append_str: Custom configuration to append to this route (optional) + * ipv6: + + * destination: IPv6 prefix + * nexthop: IPv6 nexthop address + * other options are the same as ipv4 + * extroute_ospfv3: * vrfs: @@ -170,16 +209,44 @@ Can contain the following dictionaries with specified keys: * peer_ipv4: IPv4 address of peer * route_map_in: Route-map to filter incoming routes * route_map_out: Route-map to filter outgoing routes + * ebgp_multihop: Configure eBGP multihop/TTL security, integer 1-255 + * bfd: Set to true to enable Bidirectional Forward Detection (BFD) + * graceful_restart: Set to true to enable capability graceful restart + * next_hop_self: Set to true to always advertise this router's address as the BGP next hop + * maximum_routes: Maximum routes to receive from peer, integer 0-4294967294 + * update_source: Specify local source interface for the BGP session + * auth_string: String used to calculate MD5 hash for authentication (password) * description: Description of remote peer (optional, defaults to "undefined") * cli_append_str: Custom configuration to append to this peer (optional) * neighbor_v6: - * peer_as: AS number the remote peer * peer_ipv6: IPv6 address of peer - * route_map_in: Route-map to filter incoming routes - * route_map_out: Route-map to filter outgoing routes - * description: Description of remote peer (optional, defaults to "undefined") - * cli_append_str: Custom configuration to append to this peer (optional) + * other options are the same as neighbor_v4 + +routing.yml examples: + +:: + + --- + extroute_bgp: + vrfs: + - name: OUTSIDE + local_as: 64667 + neighbor_v4: + - peer_ipv4: 10.0.255.1 + peer_as: 64666 + route_map_in: fw-lab-in + route_map_out: default-only-out + description: "fw-lab" + bfd: true + graceful_restart: true + extroute_static: + vrfs: + - name: MGMT + ipv4: + - destination: 172.12.0.0/24 + nexthop: 10.0.254.1 + name: cnaas-mgmt vxlans.yml: @@ -191,9 +258,78 @@ name is the dictionary key and dictionaly values are: * vlan_id: VLAN ID, 1-4095 * vlan_name: VLAN name, single word/no spaces, max 31 characters * ipv4_gw: IPv4 address with CIDR netmask, ex: 192.168.0.1/24. Optional. + * ipv6_gw: IPv6 address, ex: fe80::1. Optional. * groups: List of group names where this VXLAN/VLAN should be provisioned. If you select an access switch the parent dist switch should be automatically provisioned. +interfaces.yml: + +For dist and core devices interfaces are configured in YAML files. The +interface configuration can either be done per device, or per device model. +If there is a device specific folder under devices/ then the model +interface settings will be ignored. Model specific YAML files +should be named like the device model as listed in the devices API, but in +all lower-case and with all whitespaces replaced with underscore ("_"). + +Keys for interfaces.yml or interfaces_.yml: + +* interfaces: List of dicctionaries with keys: + + * name: Interface name, like "Ethernet1" + * ifclass: Interface class, one of: downlink, fabric, custom + * config: Optional. Raw CLI config used in case "custom" ifclass was selected + +The "downlink" ifclass is used on DIST devices to specify that this interface +is used to connect access devices. The "fabric" ifclass is used to specify that +this interface is used to connect DIST or CORE devices with each other to form +the switch (vxlan) fabric. Linknet data will only be configured on interfaces +specified as "fabric". If no linknet data is available in the database then +the fabric interface will be configured for ZTP of DIST/CORE devices by +providing DHCP (relay) access. + +base_system.yml: + +Contains base system settings like: + +- ntp_servers +- snmp_servers +- dns_servers +- syslog_servers +- dhcp_relays +- internal_vlans + +Example of base_system.yml: + +:: + + --- + ntp_servers: + - host: 10.255.0.1 + - host: 10.255.0.2 + snmp_servers: + - host: 10.255.0.11 + dns_servers: + - host: 10.255.0.1 + - host: 10.255.0.2 + syslog_servers: + - host: 10.255.0.21 + - host: 10.255.0.22 + dhcp_relays: + - host: 10.255.1.1 + - host: 10.255.1.2 + internal_vlans: + vlan_id_low: 3006 + vlan_id_high: 4094 + + +syslog_servers and radius_severs can optionally have the key "port" specified +to indicate a non-defalut layer4 (TCP/UDP) port number. + +internal_vlans can optionally be specified if you want to manually define +the range of internal VLANs on L3 switches. You can also specify the option +"allocation_order" under internal_vlans which is a custom string that defaults +to "ascending". If internal_vlans is specified then a collision check will +be performed for any defined vlan_ids in vxlans settings. etc --- diff --git a/requirements.txt b/requirements.txt index 80bc8299..542c0cec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,30 @@ -alembic==1.4.2 +alembic==1.4.3 APScheduler==3.6.3 -coverage==5.2.1 -Flask-Cors==3.0.8 -Flask-JWT-Extended==3.24.1 +coverage==5.3 +Flask-Cors==3.0.9 +Flask-JWT-Extended==3.25.0 flask-restx==0.2.0 -Flask-SocketIO==4.3.1 -gevent==20.6.2 -GitPython==3.1.7 -mypy==0.782 +Flask-SocketIO==5.0.0 +gevent==20.9.0 +GitPython==3.1.11 +mypy==0.790 mypy-extensions==0.4.3 -nornir==2.4.0 -napalm==3.1.0 +nornir==3.0.0 +nornir-jinja2==0.1.1 +nornir-napalm==0.1.1 +nornir-netmiko==0.1.1 +nornir-utils==0.1.1 +napalm==3.2.0 nose==1.3.7 pluggy==0.13.1 -psycopg2==2.8.5 -psycopg2-binary==2.8.5 +psycopg2==2.8.6 +psycopg2-binary==2.8.6 redis==3.5.3 redis-lru==0.1.0 -Sphinx==3.2.1 -SQLAlchemy==1.3.19 +Sphinx==3.3.1 +SQLAlchemy==1.3.20 sqlalchemy-stubs==0.3 SQLAlchemy-Utils==0.36.8 -pydantic==1.6.1 +pydantic==1.7.3 Werkzeug==1.0.1 -greenlet==0.4.16 +greenlet==0.4.17 diff --git a/src/cnaas_nms/api/app.py b/src/cnaas_nms/api/app.py index a51be805..7f3fa73c 100644 --- a/src/cnaas_nms/api/app.py +++ b/src/cnaas_nms/api/app.py @@ -16,7 +16,8 @@ from cnaas_nms.tools.log import get_logger from cnaas_nms.api.device import device_api, devices_api, device_init_api, \ - device_syncto_api, device_discover_api, device_update_facts_api + device_initcheck_api, device_syncto_api, device_discover_api, \ + device_update_facts_api, device_update_interfaces_api, device_cert_api from cnaas_nms.api.linknet import api as links_api from cnaas_nms.api.firmware import api as firmware_api from cnaas_nms.api.interface import api as interfaces_api @@ -97,9 +98,12 @@ def handle_error(self, e): api.add_namespace(device_api) api.add_namespace(devices_api) api.add_namespace(device_init_api) +api.add_namespace(device_initcheck_api) api.add_namespace(device_syncto_api) api.add_namespace(device_discover_api) api.add_namespace(device_update_facts_api) +api.add_namespace(device_update_interfaces_api) +api.add_namespace(device_cert_api) api.add_namespace(links_api) api.add_namespace(firmware_api) api.add_namespace(interfaces_api) diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py index 63839850..c61f8fee 100644 --- a/src/cnaas_nms/api/device.py +++ b/src/cnaas_nms/api/device.py @@ -10,6 +10,8 @@ import cnaas_nms.confpush.init_device import cnaas_nms.confpush.sync_devices import cnaas_nms.confpush.underlay +import cnaas_nms.confpush.get +import cnaas_nms.confpush.update from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector from cnaas_nms.api.generic import build_filter, empty_result from cnaas_nms.db.device import Device, DeviceState, DeviceType @@ -32,6 +34,8 @@ prefix='/api/{}'.format(__api_version__)) device_init_api = Namespace('device_init', description='API for init devices', prefix='/api/{}'.format(__api_version__)) +device_initcheck_api = Namespace('device_initcheck', description='API for init check of devices', + prefix='/api/{}'.format(__api_version__)) device_syncto_api = Namespace('device_syncto', description='API to sync devices', prefix='/api/{}'.format(__api_version__)) device_discover_api = Namespace('device_discover', description='API to discover devices', @@ -39,6 +43,11 @@ device_update_facts_api = Namespace('device_update_facts', description='API to update facts about devices', prefix='/api/{}'.format(__api_version__)) +device_update_interfaces_api = Namespace('device_update_interfaces', + description='API to update/scan device interfaces', + prefix='/api/{}'.format(__api_version__)) +device_cert_api = Namespace('device_cert', description='API to handle device certificates', + prefix='/api/{}'.format(__api_version__)) device_model = device_api.model('device', { @@ -63,6 +72,10 @@ 'hostname': fields.String(required=False), 'device_type': fields.String(required=False)}) +device_initcheck_model = device_initcheck_api.model('device_initcheck', { + 'hostname': fields.String(required=False), + 'device_type': fields.String(required=False)}) + device_discover_model = device_discover_api.model('device_discover', { 'ztp_mac': fields.String(required=True), 'dhcp_ip': fields.String(required=True)}) @@ -79,7 +92,13 @@ }) device_update_facts_model = device_syncto_api.model('device_update_facts', { - 'hostname': fields.String(required=False), + 'hostname': fields.String(required=True), +}) + +device_update_interfaces_model = device_syncto_api.model('device_update_interfaces', { + 'hostname': fields.String(required=True), + 'replace': fields.Boolean(required=False), + 'delete_all': fields.Boolean(required=False), }) device_restore_model = device_api.model('device_restore', { @@ -92,6 +111,18 @@ 'full_config': fields.String(required=True), }) +device_cert_model = device_syncto_api.model('device_cert', { + 'hostname': fields.String(required=False, + description="Device hostname", + example="myhostname"), + 'group': fields.String(required=False, + description="Device group", + example="mygroup"), + 'action': fields.String(required=True, + description="Action to execute, one of: RENEW", + example="RENEW") +}) + class DeviceByIdApi(Resource): @jwt_required @@ -235,65 +266,207 @@ class DeviceInitApi(Resource): @device_init_api.expect(device_init_model) def post(self, device_id: int): """ Init a device """ - if not isinstance(device_id, int): - return empty_result(status='error', data="'device_id' must be an integer"), 400 - json_data = request.get_json() + try: + job_kwargs = self.arg_check(device_id, json_data) + except ValueError as e: + return empty_result(status='error', data=str(e)), 400 + + # If device init is already in progress, reschedule a new step2 (connectivity check) + # instead of trying to restart initialization + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() + if dev and dev.state == DeviceState.INIT and \ + dev.management_ip and dev.device_type is not DeviceType.UNKNOWN: + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.init_device:init_device_step2', + when=1, + scheduled_by=get_jwt_identity(), + kwargs={'device_id': device_id, 'iteration': 1}) + + logger.info("Re-scheduled init step 2 for {} as job # {}".format( + device_id, job_id + )) + res = empty_result(data=f"Re-scheduled init step 2 for device_id { device_id }") + res['job_id'] = job_id + return res + + if job_kwargs['device_type'] == DeviceType.ACCESS.name: + del job_kwargs['device_type'] + del job_kwargs['neighbors'] + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.init_device:init_access_device_step1', + when=1, + scheduled_by=get_jwt_identity(), + kwargs=job_kwargs) + elif job_kwargs['device_type'] in [DeviceType.CORE.name, DeviceType.DIST.name]: + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.init_device:init_fabric_device_step1', + when=1, + scheduled_by=get_jwt_identity(), + kwargs=job_kwargs) + else: + return empty_result(status='error', data="Unsupported 'device_type' provided"), 400 + + res = empty_result(data=f"Scheduled job to initialize device_id { device_id }") + res['job_id'] = job_id + + return res + + @classmethod + def arg_check(cls, device_id: int, json_data: dict) -> dict: + parsed_args = { + 'device_id': device_id + } + if not isinstance(device_id, int): + raise ValueError("'device_id' must be an integer") if 'hostname' not in json_data: - return empty_result(status='error', data="POST data must include new 'hostname'"), 400 + raise ValueError("POST data must include new 'hostname'") else: if not Device.valid_hostname(json_data['hostname']): - return empty_result( - status='error', - data='Provided hostname is not valid'), 400 + raise ValueError("Provided hostname is not valid") else: - new_hostname = json_data['hostname'] + parsed_args['new_hostname'] = json_data['hostname'] if 'device_type' not in json_data: - return empty_result(status='error', data="POST data must include 'device_type'"), 400 + raise ValueError("POST data must include 'device_type'") else: try: device_type = str(json_data['device_type']).upper() except Exception: - return empty_result(status='error', data="'device_type' must be a string"), 400 + raise ValueError("'device_type' must be a string") - if not DeviceType.has_name(device_type): - return empty_result(status='error', data="Invalid 'device_type' provided"), 400 - - job_kwargs = { - 'device_id': device_id, - 'new_hostname': new_hostname - } + if DeviceType.has_name(device_type): + parsed_args['device_type'] = device_type + else: + raise ValueError("Invalid 'device_type' provided") if 'mlag_peer_id' in json_data or 'mlag_peer_hostname' in json_data: if 'mlag_peer_id' not in json_data or 'mlag_peer_hostname' not in json_data: - return empty_result( - status='error', - data="Both 'mlag_peer_id' and 'mlag_peer_hostname' must be specified"), 400 + raise ValueError("Both 'mlag_peer_id' and 'mlag_peer_hostname' must be specified") if not isinstance(json_data['mlag_peer_id'], int): - return empty_result(status='error', data="'mlag_peer_id' must be an integer"), 400 + raise ValueError("'mlag_peer_id' must be an integer") if not Device.valid_hostname(json_data['mlag_peer_hostname']): + raise ValueError("Provided 'mlag_peer_hostname' is not valid") + parsed_args['mlag_peer_id'] = json_data['mlag_peer_id'] + parsed_args['mlag_peer_new_hostname'] = json_data['mlag_peer_hostname'] + + if 'neighbors' in json_data and json_data['neighbors'] is not None: + if isinstance(json_data['neighbors'], list): + for neighbor in json_data['neighbors']: + if not Device.valid_hostname(neighbor): + raise ValueError("Invalid hostname specified in neighbor list") + parsed_args['neighbors'] = json_data['neighbors'] + else: + raise ValueError("Neighbors must be specified as either a list of hostnames," + "an empty list, or not specified at all") + else: + parsed_args['neighbors'] = None + + return parsed_args + + +class DeviceInitCheckApi(Resource): + @jwt_required + @device_init_api.expect(device_initcheck_model) + def post(self, device_id: int): + """Perform init check on a device""" + json_data = request.get_json() + ret = {} + try: + parsed_args = DeviceInitApi.arg_check(device_id, json_data) + target_devtype = DeviceType[parsed_args['device_type']] + target_hostname = parsed_args['new_hostname'] + mlag_peer_target_hostname: Optional[str] = None + mlag_peer_id: Optional[int] = None + mlag_peer_dev: Optional[Device] = None + if 'mlag_peer_id' in parsed_args and 'mlag_peer_new_hostname' in parsed_args: + mlag_peer_target_hostname = parsed_args['mlag_peer_new_hostname'] + mlag_peer_id = parsed_args['mlag_peer_id'] + except ValueError as e: + return empty_result(status='error', + data="Error parsing arguments: {}".format(e)), 400 + + with sqla_session() as session: + try: + dev = cnaas_nms.confpush.init_device.pre_init_checks(session, device_id) + except ValueError as e: + return empty_result(status='error', + data="ValueError in pre_init_checks: {}".format(e)), 400 + except Exception as e: + return empty_result(status='error', + data="Exception in pre_init_checks: {}".format(e)), 500 + + if mlag_peer_id: + try: + mlag_peer_dev = cnaas_nms.confpush.init_device.pre_init_checks( + session, mlag_peer_id) + except ValueError as e: + return empty_result(status='error', + data="ValueError in pre_init_checks: {}".format(e)), 400 + except Exception as e: + return empty_result(status='error', + data="Exception in pre_init_checks: {}".format(e)), 500 + + try: + ret['linknets'] = cnaas_nms.confpush.update.update_linknets( + session, + hostname=dev.hostname, + devtype=target_devtype, + ztp_hostname=target_hostname, + dry_run=True + ) + if mlag_peer_dev: + ret['linknets'] += cnaas_nms.confpush.update.update_linknets( + session, + hostname=mlag_peer_dev.hostname, + devtype=target_devtype, + ztp_hostname=mlag_peer_target_hostname, + dry_run=True + ) + ret['linknets_compatible'] = True + except ValueError as e: + ret['linknets_compatible'] = False + ret['linknets_error'] = str(e) + except Exception as e: + return empty_result(status='error', + data="Exception in update_linknets: {}".format(e)), 500 + + try: + if 'linknets' in ret: + ret['neighbors'] = cnaas_nms.confpush.init_device.pre_init_check_neighbors( + session, dev, target_devtype, + ret['linknets'], parsed_args['neighbors'], mlag_peer_dev) + ret['neighbors_compatible'] = True + else: + ret['neighbors_compatible'] = False + ret['neighbors_error'] = "No linknets found" + except (ValueError, cnaas_nms.confpush.init_device.InitVerificationError) as e: + ret['neighbors_compatible'] = False + ret['neighbors_error'] = str(e) + except Exception as e: return empty_result( status='error', - data="Provided 'mlag_peer_hostname' is not valid"), 400 - job_kwargs['mlag_peer_id'] = json_data['mlag_peer_id'] - job_kwargs['mlag_peer_new_hostname'] = json_data['mlag_peer_hostname'] + data="Exception in pre_init_check_neighbors: {}".format(e)), 500 - if device_type == DeviceType.ACCESS.name: - scheduler = Scheduler() - job_id = scheduler.add_onetime_job( - 'cnaas_nms.confpush.init_device:init_access_device_step1', - when=1, - scheduled_by=get_jwt_identity(), - kwargs=job_kwargs) + if mlag_peer_dev: + try: + ret['mlag_compatible'] = mlag_peer_dev.hostname in ret['neighbors'] + except Exception: + ret['mlag_compatible'] = False + + ret['parsed_args'] = parsed_args + if mlag_peer_id and not ret['mlag_compatible']: + ret['compatible'] = False + if ret['linknets_compatible'] and ret['neighbors_compatible']: + ret['compatible'] = True else: - return empty_result(status='error', data="Unsupported 'device_type' provided"), 400 - - res = empty_result(data=f"Scheduled job to initialize device_id { device_id }") - res['job_id'] = job_id - - return res + ret['compatible'] = False + return empty_result(data=ret) class DeviceDiscoverApi(Resource): @@ -366,7 +539,7 @@ def post(self): status='error', data=f"Hostname '{hostname}' not found or is not a managed device" ), 400 - kwargs['hostname'] = hostname + kwargs['hostnames'] = [hostname] what = hostname elif 'device_type' in json_data: devtype_str = str(json_data['device_type']).upper() @@ -394,7 +567,7 @@ def post(self): else: return empty_result( status='error', - data=f"No devices to synchronize was specified" + data=f"No devices to synchronize were specified" ), 400 scheduler = Scheduler() @@ -465,6 +638,97 @@ def post(self): return resp +class DeviceUpdateInterfacesApi(Resource): + @jwt_required + @device_update_interfaces_api.expect(device_update_interfaces_model) + def post(self): + """Update/scan interfaces of device""" + json_data = request.get_json() + kwargs: dict = { + "replace": False, + "delete_all": False, + "mlag_peer_hostname": None + } + + total_count: Optional[int] = None + + if 'hostname' in json_data: + hostname = str(json_data['hostname']) + if not Device.valid_hostname(hostname): + return empty_result( + status='error', + data=f"Hostname '{hostname}' is not a valid hostname" + ), 400 + with sqla_session() as session: + dev: Device = session.query(Device). \ + filter(Device.hostname == hostname).one_or_none() + if not dev or (dev.state != DeviceState.MANAGED and + dev.state != DeviceState.UNMANAGED): + return empty_result( + status='error', + data=f"Hostname '{hostname}' not found or is in invalid state" + ), 400 + if dev.device_type != DeviceType.ACCESS: + return empty_result( + status='error', + data=f"Only devices of type ACCESS has interface database to update" + ), 400 + kwargs['hostname'] = hostname + total_count = 1 + else: + return empty_result( + status='error', + data="No target to be updated was specified" + ), 400 + + if 'mlag_peer_hostname' in json_data: + mlag_peer_hostname = str(json_data['mlag_peer_hostname']) + if not Device.valid_hostname(mlag_peer_hostname): + return empty_result( + status='error', + data=f"Hostname '{mlag_peer_hostname}' is not a valid hostname" + ), 400 + with sqla_session() as session: + dev: Device = session.query(Device). \ + filter(Device.hostname == mlag_peer_hostname).one_or_none() + if not dev or (dev.state != DeviceState.MANAGED and + dev.state != DeviceState.UNMANAGED): + return empty_result( + status='error', + data=f"Hostname '{mlag_peer_hostname}' not found or is in invalid state" + ), 400 + if dev.device_type != DeviceType.ACCESS: + return empty_result( + status='error', + data=f"Only devices of type ACCESS has interface database to update" + ), 400 + kwargs['mlag_peer_hostname'] = mlag_peer_hostname + + if 'replace' in json_data and isinstance(json_data['replace'], bool) \ + and json_data['replace']: + kwargs['replace'] = True + + if 'delete_all' in json_data and isinstance(json_data['delete_all'], bool) \ + and json_data['delete_all']: + kwargs['delete_all'] = True + + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.update:update_interfacedb', + when=1, + scheduled_by=get_jwt_identity(), + kwargs=kwargs) + + res = empty_result(data=f"Scheduled job to update interfaces for {hostname}") + res['job_id'] = job_id + + resp = make_response(json.dumps(res), 200) + if total_count: + resp.headers['X-Total-Count'] = total_count + resp.headers['Content-Type'] = "application/json" + return resp + + class DeviceConfigApi(Resource): @jwt_required def get(self, hostname: str): @@ -642,6 +906,80 @@ def post(self, hostname: str): return res, 200 +class DeviceCertApi(Resource): + @jwt_required + @device_api.expect(device_cert_model) + def post(self): + """Execute certificate related actions on device""" + json_data = request.get_json() + # default args + kwargs: dict = {} + + if 'action' in json_data and isinstance(json_data['action'], str): + action = json_data['action'].upper() + else: + return empty_result( + status='error', + data=f"Required field 'action' was not specified" + ), 400 + + if 'comment' in json_data and isinstance(json_data['comment'], str): + kwargs['job_comment'] = json_data['comment'] + if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'], str): + kwargs['job_ticket_ref'] = json_data['ticket_ref'] + + total_count: Optional[int] = None + nr = cnaas_init() + + if 'hostname' in json_data: + hostname = str(json_data['hostname']) + if not Device.valid_hostname(hostname): + return empty_result( + status='error', + data=f"Hostname '{hostname}' is not a valid hostname" + ), 400 + _, total_count, _ = inventory_selector(nr, hostname=hostname) + if total_count != 1: + return empty_result( + status='error', + data=f"Hostname '{hostname}' not found or is not a managed device" + ), 400 + kwargs['hostname'] = hostname + elif 'group' in json_data: + group_name = str(json_data['group']) + if group_name not in get_groups(): + return empty_result(status='error', data='Could not find a group with name {}'.format(group_name)) + kwargs['group'] = group_name + _, total_count, _ = inventory_selector(nr, group=group_name) + else: + return empty_result( + status='error', + data=f"No devices were specified" + ), 400 + + if action == 'RENEW': + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.cert:renew_cert', + when=1, + scheduled_by=get_jwt_identity(), + kwargs=kwargs) + + res = empty_result(data=f"Scheduled job to renew certificates") + res['job_id'] = job_id + + resp = make_response(json.dumps(res), 200) + if total_count: + resp.headers['X-Total-Count'] = total_count + resp.headers['Content-Type'] = "application/json" + return resp + else: + return empty_result( + status='error', + data=f"Unknown action specified: {action}" + ), 400 + + # Devices device_api.add_resource(DeviceByIdApi, '/') device_api.add_resource(DeviceByHostnameApi, '/') @@ -651,7 +989,10 @@ def post(self, hostname: str): device_api.add_resource(DeviceApi, '') devices_api.add_resource(DevicesApi, '') device_init_api.add_resource(DeviceInitApi, '/') +device_initcheck_api.add_resource(DeviceInitCheckApi, '/') device_discover_api.add_resource(DeviceDiscoverApi, '') device_syncto_api.add_resource(DeviceSyncApi, '') device_update_facts_api.add_resource(DeviceUpdateFactsApi, '') +device_update_interfaces_api.add_resource(DeviceUpdateInterfacesApi, '') +device_cert_api.add_resource(DeviceCertApi, '') # device//current_config diff --git a/src/cnaas_nms/api/firmware.py b/src/cnaas_nms/api/firmware.py index 58d4af09..afc6e61d 100644 --- a/src/cnaas_nms/api/firmware.py +++ b/src/cnaas_nms/api/firmware.py @@ -3,8 +3,9 @@ import requests from datetime import datetime +from typing import Optional -from flask import request +from flask import request, make_response from flask_restx import Resource, Namespace, fields from flask_jwt_extended import jwt_required, get_jwt_identity @@ -14,6 +15,9 @@ from cnaas_nms.tools.log import get_logger from cnaas_nms.tools.get_apidata import get_apidata from cnaas_nms.version import __api_version__ +from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector +from cnaas_nms.db.device import Device +from cnaas_nms.db.settings import get_groups logger = get_logger() @@ -36,6 +40,8 @@ 'group': fields.String(required=False), 'hostname': fields.String(required=False), 'pre_flight': fields.Boolean(required=False), + 'post_flight': fields.Boolean(required=False), + 'post_wattime': fields.Integer(required=False), 'reboot': fields.Boolean(required=False)}) @@ -240,6 +246,20 @@ def post(self): return empty_result(status='error', data='pre_flight should be a boolean') + if 'post_flight' in json_data: + if isinstance(json_data['post_flight'], bool): + kwargs['post_flight'] = json_data['post_flight'] + else: + return empty_result(status='error', + data='post_flight should be a boolean') + + if 'post_waittime' in json_data: + if isinstance(json_data['post_waittime'], int): + kwargs['post_waittime'] = json_data['post_waittime'] + else: + return empty_result(status='error', + data='post_waittime should be an integer') + if 'filename' in json_data: if isinstance(json_data['filename'], str): kwargs['filename'] = json_data['filename'] @@ -247,31 +267,52 @@ def post(self): return empty_result(status='error', data='filename should be a string') - if 'group' in json_data: - if isinstance(json_data['group'], str): - kwargs['group'] = json_data['group'] - else: - return empty_result(status='error', - data='group should be a string') + total_count: Optional[int] = None + nr = cnaas_init() if 'hostname' in json_data: - if isinstance(json_data['hostname'], str): - kwargs['hostname'] = json_data['hostname'] - else: - return empty_result(status='error', - data='hostname should be a string') + hostname = str(json_data['hostname']) + if not Device.valid_hostname(hostname): + return empty_result( + status='error', + data=f"Hostname '{hostname}' is not a valid hostname" + ), 400 + _, total_count, _ = inventory_selector(nr, hostname=hostname) + if total_count != 1: + return empty_result( + status='error', + data=f"Hostname '{hostname}' not found or is not a managed device" + ), 400 + kwargs['hostname'] = hostname + elif 'group' in json_data: + group_name = str(json_data['group']) + if group_name not in get_groups(): + return empty_result(status='error', data='Could not find a group with name {}'.format(group_name)) + kwargs['group'] = group_name + _, total_count, _ = inventory_selector(nr, group=group_name) + kwargs['group'] = group_name + else: + return empty_result( + status='error', + data=f"No devices to upgrade were specified" + ), 400 + + if 'comment' in json_data and isinstance(json_data['comment'], str): + kwargs['job_comment'] = json_data['comment'] + if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'], str): + kwargs['job_ticket_ref'] = json_data['ticket_ref'] if 'start_at' in json_data: try: time_start = datetime.strptime(json_data['start_at'], date_format) - time_now = datetime.now() + time_now = datetime.utcnow() if time_start < time_now: return empty_result(status='error', data='start_at must be in the future') time_diff = time_start - time_now - seconds = time_diff.seconds + seconds = int(time_diff.total_seconds()) except Exception as e: logger.exception(f'Exception when scheduling job: {e}') return empty_result(status='error', @@ -286,7 +327,11 @@ def post(self): res = empty_result(data='Scheduled job to upgrade devices') res['job_id'] = job_id - return res + resp = make_response(json.dumps(res), 200) + if total_count: + resp.headers['X-Total-Count'] = total_count + resp.headers['Content-Type'] = "application/json" + return resp # Firmware diff --git a/src/cnaas_nms/api/groups.py b/src/cnaas_nms/api/groups.py index 1b2977a4..087d906a 100644 --- a/src/cnaas_nms/api/groups.py +++ b/src/cnaas_nms/api/groups.py @@ -1,11 +1,12 @@ from typing import List, Optional +import re from flask_restx import Resource, Namespace from flask_jwt_extended import jwt_required -from cnaas_nms.db.device import Device +from cnaas_nms.db.device import Device, DeviceState from cnaas_nms.api.generic import empty_result -from cnaas_nms.db.settings import get_groups +from cnaas_nms.db.settings import get_groups, get_group_regex from cnaas_nms.db.session import sqla_session from cnaas_nms.version import __api_version__ @@ -31,6 +32,29 @@ def groups_populate(group_name: Optional[str] = None): return tmpgroups +def groups_osversion_populate(group_name: str): + os_versions: dict = {} + group_regex = get_group_regex(group_name) + if group_regex: + group_regex_p = re.compile(group_regex) + else: + raise ValueError("Could not find group {}".format(group_name)) + + with sqla_session() as session: + devices: List[Device] = session.query(Device).\ + filter(Device.state == DeviceState.MANAGED).\ + order_by(Device.hostname.asc()).all() + for dev in devices: + if not dev.os_version: + continue + if re.match(group_regex_p, dev.hostname): + if dev.os_version in os_versions: + os_versions[dev.os_version].append(dev.hostname) + else: + os_versions[dev.os_version] = [dev.hostname] + return {group_name: os_versions} + + class GroupsApi(Resource): @jwt_required def get(self): @@ -40,14 +64,35 @@ def get(self): return empty_result(status='success', data=result) -class GroupsApiById(Resource): +class GroupsApiByName(Resource): @jwt_required def get(self, group_name): - """ Get a single group by ID """ + """ Get a single group by name """ tmpgroups = groups_populate(group_name) result = {'groups': tmpgroups} return empty_result(status='success', data=result) +class GroupsApiByNameOsversion(Resource): + @jwt_required + def get(self, group_name): + """Get os version of all devices in a group""" + try: + group_os_versions = groups_osversion_populate(group_name) + except ValueError as e: + return empty_result( + status='error', + data="Exception while getting group {}: {}".format(group_name, str(e)) + ), 404 + except Exception as e: + return empty_result( + status='error', + data="Exception while getting group {}: {}".format(group_name, str(e)) + ), 500 + result = {'groups': group_os_versions} + return empty_result(status='success', data=result) + + api.add_resource(GroupsApi, '') -api.add_resource(GroupsApiById, '/') +api.add_resource(GroupsApiByName, '/') +api.add_resource(GroupsApiByNameOsversion, '//os_version') diff --git a/src/cnaas_nms/api/interface.py b/src/cnaas_nms/api/interface.py index 1d90ba96..1f4629bd 100644 --- a/src/cnaas_nms/api/interface.py +++ b/src/cnaas_nms/api/interface.py @@ -31,8 +31,12 @@ def get(self, hostname): result['data']['hostname'] = dev.hostname intfs = session.query(Interface).filter(Interface.device == dev).all() intf: Interface + interfaces = [] for intf in intfs: - result['data']['interfaces'].append(intf.as_dict()) + ifdict = intf.as_dict() + ifdict['indexnum'] = Interface.interface_index_num(ifdict['name']) + interfaces.append(ifdict) + result['data']['interfaces'] = sorted(interfaces, key=lambda i: i['indexnum']) return result @jwt_required diff --git a/src/cnaas_nms/api/jobs.py b/src/cnaas_nms/api/jobs.py index a70d3993..2a95d6dc 100644 --- a/src/cnaas_nms/api/jobs.py +++ b/src/cnaas_nms/api/jobs.py @@ -1,14 +1,17 @@ import json +import time + from flask import request, make_response from flask_restx import Resource, Namespace, fields -from flask_jwt_extended import jwt_required +from flask_jwt_extended import jwt_required, get_jwt_identity from sqlalchemy import func from cnaas_nms.api.generic import empty_result, build_filter -from cnaas_nms.db.job import Job +from cnaas_nms.db.job import Job, JobStatus from cnaas_nms.db.joblock import Joblock from cnaas_nms.db.session import sqla_session from cnaas_nms.version import __api_version__ +from cnaas_nms.scheduler.scheduler import Scheduler job_api = Namespace('job', description='API for handling jobs', @@ -55,7 +58,51 @@ def get(self, job_id): if job: return empty_result(data={'jobs': [job.as_dict()]}) else: - return empty_result(status='error', data="No job with id {} found".format(job_id)), 400 + return empty_result(status='error', + data="No job with id {} found".format(job_id)), 400 + + @jwt_required + def put(self, job_id): + json_data = request.get_json() + if 'action' not in json_data: + return empty_result(status='error', data="Action must be specified"), 400 + + with sqla_session() as session: + job = session.query(Job).filter(Job.id == job_id).one_or_none() + if not job: + return empty_result(status='error', + data="No job with id {} found".format(job_id)), 400 + job_status = job.status + + action = str(json_data['action']).upper() + if action == 'ABORT': + allowed_jobstates = [JobStatus.SCHEDULED, JobStatus.RUNNING] + if job_status not in allowed_jobstates: + return empty_result( + status='error', + data="Job id {} is in state {}, must be {} to abort".format( + job_id, job_status, (" or ".join([x.name for x in allowed_jobstates])) + )), 400 + abort_reason = "Aborted via API call" + if 'abort_reason' in json_data and isinstance(json_data['abort_reason'], str): + abort_reason = json_data['abort_reason'][:255] + + abort_reason += " (aborted by {})".format(get_jwt_identity()) + + if job_status == JobStatus.SCHEDULED: + scheduler = Scheduler() + scheduler.remove_scheduled_job(job_id=job_id, abort_message=abort_reason) + time.sleep(2) + elif job_status == JobStatus.RUNNING: + with sqla_session() as session: + job = session.query(Job).filter(Job.id == job_id).one_or_none() + job.status = JobStatus.ABORTING + + with sqla_session() as session: + job = session.query(Job).filter(Job.id == job_id).one_or_none() + return empty_result(data={"jobs": [job.as_dict()]}) + else: + return empty_result(status='error', data="Unknown action: {}".format(action)), 400 class JobLockApi(Resource): diff --git a/src/cnaas_nms/api/linknet.py b/src/cnaas_nms/api/linknet.py index a99de46c..916e746c 100644 --- a/src/cnaas_nms/api/linknet.py +++ b/src/cnaas_nms/api/linknet.py @@ -1,11 +1,12 @@ from flask import request from flask_restx import Resource, Namespace, fields from flask_jwt_extended import jwt_required +from ipaddress import IPv4Network from cnaas_nms.api.generic import empty_result from cnaas_nms.db.session import sqla_session from cnaas_nms.db.linknet import Linknet -from cnaas_nms.db.device import Device +from cnaas_nms.db.device import Device, DeviceType from cnaas_nms.confpush.underlay import find_free_infra_linknet from cnaas_nms.version import __api_version__ @@ -42,11 +43,15 @@ def post(self): if 'device_a' in json_data: if not Device.valid_hostname(json_data['device_a']): errors.append("Invalid hostname specified for device_a") + else: + hostname_a = json_data['device_a'] else: errors.append("Required field hostname_a not found") if 'device_b' in json_data: if not Device.valid_hostname(json_data['device_b']): errors.append("Invalid hostname specified for device_b") + else: + hostname_b = json_data['device_b'] else: errors.append("Required field hostname_b not found") if 'device_a_port' not in json_data: @@ -54,15 +59,50 @@ def post(self): if 'device_b_port' not in json_data: errors.append("Required field device_b_port not found") + new_prefix = None + if 'prefix' in json_data: + if json_data['prefix']: + try: + new_prefix = IPv4Network(json_data['prefix']) + except Exception as e: + errors.append("Invalid prefix: {}".format(e)) + if errors: return empty_result(status='error', data=errors), 400 with sqla_session() as session: + dev_a: Device = session.query(Device).\ + filter(Device.hostname == hostname_a).one_or_none() + if not dev_a: + return empty_result( + status='error', + data=f"Hostname '{hostname_a}' not found or is in invalid state" + ), 400 + + dev_b: Device = session.query(Device). \ + filter(Device.hostname == hostname_b).one_or_none() + if not dev_b: + return empty_result( + status='error', + data=f"Hostname '{hostname_b}' not found or is in invalid state" + ), 400 + + # check if we need an ip prefix for the linknet + ip_linknet_devtypes = [DeviceType.CORE, DeviceType.DIST] + if dev_a.device_type in ip_linknet_devtypes and \ + dev_b.device_type in ip_linknet_devtypes: + if not new_prefix: + new_prefix = find_free_infra_linknet(session) + if not new_prefix: + return empty_result( + status='error', + data="Device types requires IP linknets, but no prefix could be found" + ), 400 + try: - new_prefix = find_free_infra_linknet(session) new_linknet = Linknet.create_linknet( - session, json_data['device_a'], json_data['device_a_port'], - json_data['device_b'], json_data['device_b_port'], new_prefix) + session, hostname_a, json_data['device_a_port'], + hostname_b, json_data['device_b_port'], new_prefix) session.add(new_linknet) session.commit() data = new_linknet.as_dict() diff --git a/src/cnaas_nms/api/mgmtdomain.py b/src/cnaas_nms/api/mgmtdomain.py index c61a180c..0c51eb20 100644 --- a/src/cnaas_nms/api/mgmtdomain.py +++ b/src/cnaas_nms/api/mgmtdomain.py @@ -43,12 +43,15 @@ def get(self, mgmtdomain_id): def delete(self, mgmtdomain_id): """ Remove management domain """ with sqla_session() as session: - instance = session.query(Mgmtdomain).\ + instance: Mgmtdomain = session.query(Mgmtdomain).\ filter(Mgmtdomain.id == mgmtdomain_id).one_or_none() if instance: + instance.device_a.synchronized = False + instance.device_b.synchronized = False session.delete(instance) session.commit() - return empty_result(), 204 + return empty_result(status="success", + data={"deleted_mgmtdomain": instance.as_dict()}), 200 else: return empty_result('error', "Management domain not found"), 404 @@ -79,8 +82,11 @@ def put(self, mgmtdomain_id): errors.append("Bad prefix length for management network: {}".format( prefix_len)) with sqla_session() as session: - instance = session.query(Mgmtdomain).filter(Mgmtdomain.id == mgmtdomain_id).one_or_none() + instance: Mgmtdomain = session.query(Mgmtdomain).\ + filter(Mgmtdomain.id == mgmtdomain_id).one_or_none() if instance: + instance.device_a.synchronized = False + instance.device_b.synchronized = False #TODO: auto loop through class members and match if 'vlan' in data: instance.vlan = data['vlan'] @@ -119,7 +125,7 @@ def post(self): if not Device.valid_hostname(hostname_a): errors.append(f"Invalid hostname for device_a: {hostname_a}") else: - device_a = session.query(Device).\ + device_a: Device = session.query(Device).\ filter(Device.hostname == hostname_a).one_or_none() if not device_a: errors.append(f"Device with hostname {hostname_a} not found") @@ -130,7 +136,7 @@ def post(self): if not Device.valid_hostname(hostname_b): errors.append(f"Invalid hostname for device_b: {hostname_b}") else: - device_b = session.query(Device).\ + device_b: Device = session.query(Device).\ filter(Device.hostname == hostname_b).one_or_none() if not device_b: errors.append(f"Device with hostname {hostname_b} not found") @@ -163,8 +169,11 @@ def post(self): new_mgmtd.device_b = data['device_b'] new_mgmtd.ipv4_gw = data['ipv4_gw'] new_mgmtd.vlan = data['vlan'] - result = session.add(new_mgmtd) - return empty_result(result, 200) + session.add(new_mgmtd) + session.flush() + device_a.synchronized = False + device_b.synchronized = False + return empty_result(status='success', data={"added_mgmtdomain": new_mgmtd.as_dict()}), 200 else: errors.append("Not all required inputs were found: {}".\ format(', '.join(required_keys))) diff --git a/src/cnaas_nms/api/settings.py b/src/cnaas_nms/api/settings.py index 29b98d17..d4aaff3b 100644 --- a/src/cnaas_nms/api/settings.py +++ b/src/cnaas_nms/api/settings.py @@ -25,6 +25,7 @@ def get(self): args = request.args hostname = None device_type = None + model = None if 'hostname' in args: if Device.valid_hostname(args['hostname']): hostname = args['hostname'] @@ -35,6 +36,7 @@ def get(self): filter(Device.hostname == hostname).one_or_none() if dev: device_type = dev.device_type + model = dev.model else: return empty_result('error', "Hostname not found in database"), 400 if 'device_type' in args: @@ -44,7 +46,7 @@ def get(self): return empty_result('error', "Invalid device type specified"), 400 try: - settings, settings_origin = get_settings(hostname, device_type) + settings, settings_origin = get_settings(hostname, device_type, model) except Exception as e: return empty_result('error', "Error getting settings: {}".format(str(e))), 400 @@ -55,7 +57,6 @@ class SettingsModelApi(Resource): def get(self): response = make_response(settings_root_model.schema_json()) response.headers['Content-Type'] = "application/json" - response.headers['Content-Type'] = 'application/json' return response def post(self): diff --git a/src/cnaas_nms/api/tests/data/testdata.yml b/src/cnaas_nms/api/tests/data/testdata.yml index 80070899..0c1d6a81 100644 --- a/src/cnaas_nms/api/tests/data/testdata.yml +++ b/src/cnaas_nms/api/tests/data/testdata.yml @@ -5,3 +5,6 @@ interface_uplink: 'Ethernet3' untagged_vlan: 'STUDENT' tagged_vlan_list: ['STUDENT', 'STUDENT2'] jwt_auth_token: "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJpYXQiOjE1NzEwNTk2MTgsIm5iZiI6MTU3MTA1OTYxOCwianRpIjoiNTQ2MDk2YTUtZTNmOS00NzFlLWE2NTctZWFlYTZkNzA4NmVhIiwic3ViIjoiYWRtaW4iLCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.Sfffg9oZg_Kmoq7Oe8IoTcbuagpP6nuUXOQzqJpgDfqDq_GM_4zGzt7XxByD4G0q8g4gZGHQnV14TpDer2hJXw" +initcheck_device_id: 3 +groupname: 'ALL' +managed_dist: 'eosdist1' \ No newline at end of file diff --git a/src/cnaas_nms/api/tests/test_api.py b/src/cnaas_nms/api/tests/test_api.py index 0262344d..d0d9d95d 100644 --- a/src/cnaas_nms/api/tests/test_api.py +++ b/src/cnaas_nms/api/tests/test_api.py @@ -22,7 +22,7 @@ def setUp(self): self.client = self.app.test_client() def test_get_single_device(self): - hostname = "eosdist1" + hostname = self.testdata['managed_dist'] result = self.client.get( f'/api/v1.0/devices', params={"filter[hostname]": hostname} @@ -323,6 +323,36 @@ def test_bounce_interface(self): self.assertEqual(result.status_code, 200) self.assertEqual(result.json['status'], 'success') + def test_get_groups(self): + groupname = self.testdata['groupname'] + result = self.client.get("/api/v1.0/groups") + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json['status'], 'success') + self.assertTrue(groupname in result.json['data']['groups'], + f"Group '{groupname}' not found") + self.assertGreaterEqual(len(result.json['data']['groups'][groupname]), 1, + f"No devices found in group '{groupname}'") + + def test_get_groups_osversion(self): + groupname = self.testdata['groupname'] + result = self.client.get(f"/api/v1.0/groups/{groupname}/os_version") + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json['status'], 'success') + self.assertGreaterEqual(len(result.json['data']['groups'][groupname]), 1, + f"No devices found in group '{groupname}' os_versions") + + def test_renew_cert_errors(self): + # Test invalid hostname + data = {"hostname": "...", "action": "RENEW"} + result = self.client.post('/api/v1.0/device_cert', json=data) + self.assertEqual(result.status_code, 400) + + # Test invalid action + data = {"hostname": self.testdata['managed_dist']} + result = self.client.post('/api/v1.0/device_cert', json=data) + self.assertEqual(result.status_code, 400) + self.assertTrue("action" in result.json['message'], msg="Unexpected error message") + if __name__ == '__main__': unittest.main() diff --git a/src/cnaas_nms/api/tests/test_device.py b/src/cnaas_nms/api/tests/test_device.py index 9c0d1224..4dc6de3b 100644 --- a/src/cnaas_nms/api/tests/test_device.py +++ b/src/cnaas_nms/api/tests/test_device.py @@ -24,6 +24,9 @@ def setUp(self): self.tmp_postgres = PostgresTemporaryInstance() def tearDown(self): + device_id = self.testdata['initcheck_device_id'] + self.client.put(f'/api/v1.0/device/{device_id}', + json={'state': 'MANAGED'}) self.tmp_postgres.shutdown() def test_0_add_invalid_device(self): @@ -131,6 +134,24 @@ def test_4_delete_device(self): result = self.client.delete(f'/api/v1.0/device/{device_id}') self.assertEqual(result.status_code, 200) + def test_5_initcheck_distdevice(self): + device_id = self.testdata['initcheck_device_id'] + self.client.put(f'/api/v1.0/device/{device_id}', + json={'state': 'DISCOVERED'}) + + device_data = { + "hostname": "distcheck", + "device_type": "DIST" + } + result = self.client.post(f'/api/v1.0/device_initcheck/{device_id}', + json=device_data) + self.assertEqual(result.status_code, 200) + json_data = json.loads(result.data.decode()) + self.assertEqual(json_data['data']['compatible'], False) + + self.client.put(f'/api/v1.0/device/{device_id}', + json={'state': 'MANAGED'}) + if __name__ == '__main__': unittest.main() diff --git a/src/cnaas_nms/confpush/baseconfig.py b/src/cnaas_nms/confpush/baseconfig.py deleted file mode 100644 index ae9658d4..00000000 --- a/src/cnaas_nms/confpush/baseconfig.py +++ /dev/null @@ -1,41 +0,0 @@ - -from typing import Optional - -from nornir.core.filter import F -from nornir.plugins.tasks import networking, text - -from cnaas_nms.scheduler.wrapper import job_wrapper -from cnaas_nms.confpush.nornir_helper import NornirJobResult -from cnaas_nms.db.device import DeviceType -from cnaas_nms.tools.log import get_logger - -logger = get_logger() - - -@job_wrapper -def sync_basetemplate(hostname: Optional[str] = None, - device_type: Optional[DeviceType] = None, - dry_run: bool=True, job_id: Optional[str] = None) -> NornirJobResult: - """Synchronize base system template to device or device group. - - Args: - hostname: Hostname of a single device to sync - device_type: A device group type to sync - dry_run: Set to true to only perform a NAPALM dry_run of config changes - """ - nrresult = None - - nr = cnaas_nms.confpush.nornir_helper.cnaas_init() - if hostname and isinstance(hostname, str): - nr_filtered = nr.filter(name=hostname) - elif device_type and isinstance(device_type, DeviceType): - group_name = ('T_'+device_type.value) - nr_filtered = nr.filter(F(groups__contains=group_name)) - else: - raise ValueError("hostname or device_type must be specified") - - nrresult = nr_filtered.run(task=push_basetemplate) - - return NornirJobResult( - nrresult = nrresult - ) diff --git a/src/cnaas_nms/confpush/cert.py b/src/cnaas_nms/confpush/cert.py new file mode 100644 index 00000000..8526ebab --- /dev/null +++ b/src/cnaas_nms/confpush/cert.py @@ -0,0 +1,170 @@ +import os +from typing import Optional + +from nornir_netmiko import netmiko_file_transfer, netmiko_send_command + +from cnaas_nms.scheduler.thread_data import set_thread_data +from cnaas_nms.tools.log import get_logger +from cnaas_nms.tools.get_apidata import get_apidata +from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector +from cnaas_nms.scheduler.wrapper import job_wrapper +from cnaas_nms.db.session import sqla_session +from cnaas_nms.confpush.nornir_helper import NornirJobResult +from cnaas_nms.db.device import Device +from cnaas_nms.tools.pki import generate_device_cert + + +class CopyError(Exception): + pass + + +def arista_copy_cert(task, job_id: Optional[str] = None) -> str: + set_thread_data(job_id) + logger = get_logger() + apidata = get_apidata() + + try: + key_path = os.path.join(apidata['certpath'], "{}.key".format(task.host.name)) + crt_path = os.path.join(apidata['certpath'], "{}.crt".format(task.host.name)) + except KeyError: + raise Exception("No certpath found in api.yml settings") + except Exception as e: + raise Exception("Unable to find path to cert {} for device".format(e, task.host.name)) + + if not os.path.isfile(key_path): + raise Exception("Key file {} not found".format(key_path)) + if not os.path.isfile(crt_path): + raise Exception("Cert file {} not found".format(crt_path)) + + net_connect = task.host.get_connection("netmiko", task.nornir.config) + net_connect.fast_cli = False + + res_key = task.run( + netmiko_file_transfer, + source_file=key_path, + dest_file="cnaasnms.key", + file_system="/mnt/flash", + overwrite_file=True + ) + if res_key.failed: + logger.exception(res_key.exception) + + res_crt = task.run( + netmiko_file_transfer, + source_file=crt_path, + dest_file="cnaasnms.crt", + file_system="/mnt/flash", + overwrite_file=True + ) + if res_crt.failed: + logger.exception(res_crt.exception) + + if res_key.failed or res_crt.failed: + raise CopyError("Unable to copy cert file to device: {}".format(task.host.name)) + else: + logger.debug("Certificate successfully copied to device: {}".format(task.host.name)) + + certstore_commands = [ + "copy flash:cnaasnms.crt certificate:", + "copy flash:cnaasnms.key sslkey:", + "delete flash:cnaasnms.key", + "delete flash:cnaasnms.crt" + ] + for cmd in certstore_commands: + res_certstore = task.run( + netmiko_send_command, + command_string=cmd, + enable=True + ) + if res_certstore.failed: + logger.error( + "Unable to copy cert into certstore on device: {}, command '{}' failed".format( + task.host.name, cmd + )) + raise CopyError("Unable to copy cert into certstore on device: {}". + format(task.host.name)) + + logger.debug("Certificate successfully copied to certstore on device: {}". + format(task.host.name)) + return "Cert copy successful" + + +def renew_cert_task(task, job_id: str) -> str: + set_thread_data(job_id) + logger = get_logger() + + with sqla_session() as session: + dev: Device = session.query(Device). \ + filter(Device.hostname == task.host.name).one_or_none() + ip = dev.management_ip + if not ip: + raise Exception("Device {} has no management_ip".format(task.host.name)) + + try: + generate_device_cert(task.host.name, ipv4_address=ip) + except Exception as e: + raise Exception("Could not generate certificate for device {}: {}".format( + task.host.name, e + )) + + if task.host.platform == "eos": + try: + res = task.run(task=arista_copy_cert, + job_id=job_id) + except Exception as e: + logger.exception('Exception while copying certificates: {}'.format( + str(e))) + raise e + else: + raise ValueError("Unsupported platform: {}".format(task.host.platform)) + + return "Certificate renew success for device {}".format(task.host.name) + + +@job_wrapper +def renew_cert(hostname: Optional[str] = None, + group: Optional[str] = None, + job_id: Optional[str] = None, + scheduled_by: Optional[str] = None) -> NornirJobResult: + + logger = get_logger() + nr = cnaas_init() + if hostname: + nr_filtered, dev_count, _ = inventory_selector(nr, hostname=hostname) + elif group: + nr_filtered, dev_count, _ = inventory_selector(nr, group=group) + else: + raise ValueError("Neither hostname nor group specified for renew_cert") + + device_list = list(nr_filtered.inventory.hosts.keys()) + logger.info("Device(s) selected for renew certificate ({}): {}".format( + dev_count, ", ".join(device_list) + )) + + supported_platforms = ['eos'] + # Make sure we only attempt supported devices + for device in device_list: + with sqla_session() as session: + dev: Device = session.query(Device). \ + filter(Device.hostname == device).one_or_none() + if not dev: + raise Exception('Could not find device: {}'.format(device)) + if dev.platform not in supported_platforms: + raise Exception('Unsupported device platform "{}" for device: {}'.format( + dev.platform, device)) + + try: + nrresult = nr_filtered.run(task=renew_cert_task, job_id=job_id) + except Exception as e: + logger.exception('Exception while renewing certificates: {}'.format( + str(e))) + return NornirJobResult(nrresult=nrresult) + + failed_hosts = list(nrresult.failed_hosts.keys()) + for hostname in failed_hosts: + logger.error("Certificate renew on device '{}' failed".format(hostname)) + + if nrresult.failed: + logger.error("Not all devices got new certificates") + + return NornirJobResult(nrresult=nrresult) diff --git a/src/cnaas_nms/confpush/erase.py b/src/cnaas_nms/confpush/erase.py index 04c8a2d6..76a35a9c 100644 --- a/src/cnaas_nms/confpush/erase.py +++ b/src/cnaas_nms/confpush/erase.py @@ -1,14 +1,13 @@ import cnaas_nms.confpush.nornir_helper from cnaas_nms.tools.log import get_logger -from cnaas_nms.scheduler.scheduler import Scheduler from cnaas_nms.scheduler.wrapper import job_wrapper from cnaas_nms.confpush.nornir_helper import NornirJobResult from cnaas_nms.db.session import sqla_session from cnaas_nms.db.device import DeviceType, Device -from nornir.plugins.functions.text import print_result -from nornir.plugins.tasks.networking import netmiko_send_command +from nornir_netmiko.tasks import netmiko_send_command +from nornir_utils.plugins.functions import print_result logger = get_logger() @@ -19,7 +18,6 @@ def device_erase_task(task, hostname: str) -> str: res = task.run(netmiko_send_command, command_string='enable', expect_string='.*#', name='Enable') - print_result(res) res = task.run(netmiko_send_command, command_string='write erase now', @@ -31,6 +29,19 @@ def device_erase_task(task, hostname: str) -> str: task.host.name, e)) raise Exception('Factory default device') + # Remove cnaas device certificates if they are found + try: + task.run(netmiko_send_command, + command_string='delete certificate:cnaasnms.crt', + expect_string='.*#', + name='Remove device certificate') + task.run(netmiko_send_command, + command_string='delete sslkey:cnaasnms.key', + expect_string='.*#', + name='Remove device key') + except Exception as e: + pass + try: res = task.run(netmiko_send_command, command_string='reload force', max_loops=2, diff --git a/src/cnaas_nms/confpush/firmware.py b/src/cnaas_nms/confpush/firmware.py index 2e1a29b2..c96ee856 100644 --- a/src/cnaas_nms/confpush/firmware.py +++ b/src/cnaas_nms/confpush/firmware.py @@ -1,24 +1,26 @@ +import time +from typing import Optional + +from nornir.core.task import MultiResult +from nornir.core.exceptions import NornirSubTaskError +from nornir_napalm.plugins.tasks import napalm_cli, napalm_get +from nornir_netmiko.tasks import netmiko_send_command + from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector from cnaas_nms.tools.log import get_logger from cnaas_nms.scheduler.wrapper import job_wrapper from cnaas_nms.confpush.nornir_helper import NornirJobResult from cnaas_nms.db.session import sqla_session, redis_session from cnaas_nms.db.device import DeviceType, Device +from cnaas_nms.db.job import Job +from cnaas_nms.scheduler.thread_data import set_thread_data -from nornir.plugins.functions.text import print_result -from nornir.plugins.tasks.networking import napalm_cli -from nornir.plugins.tasks.networking import netmiko_send_command -from nornir.core.task import MultiResult - -from napalm.base.exceptions import CommandErrorException -from typing import Optional - - -logger = get_logger() +class FirmwareAlreadyActiveException(Exception): + pass -def arista_pre_flight_check(task): +def arista_pre_flight_check(task, job_id: Optional[str] = None) -> str: """ NorNir task to do some basic checks before attempting to upgrade a switch. @@ -26,17 +28,21 @@ def arista_pre_flight_check(task): task: NorNir task Returns: - Nope, nothing. + String, describing the result """ - logger.info("Pre-flight check for {}".format(task.host.name)) + set_thread_data(job_id) + logger = get_logger() + with sqla_session() as session: + if Job.check_job_abort_status(session, job_id): + return "Pre-flight aborted" flash_diskspace = 'bash timeout 5 df /mnt/flash | awk \'{print $4}\'' flash_cleanup = 'bash timeout 30 ls -t /mnt/flash/*.swi | tail -n +2 | grep -v `cut -d"/" -f2 /mnt/flash/boot-config` | xargs rm -f' # Get amount of free disk space res = task.run(napalm_cli, commands=[flash_diskspace]) - if not isinstance(res, MultiResult) or len(res.result.keys()) is not 1: + if not isinstance(res, MultiResult) or len(res.result.keys()) != 1: raise Exception('Could not check free space') # Remove old firmware images if needed @@ -44,14 +50,56 @@ def arista_pre_flight_check(task): if int(free_bytes) < 2500000: logger.info('Cleaning up old firmware images on {}'.format(task.host.name)) res = task.run(napalm_cli, commands=[flash_cleanup]) - print_result(res) else: logger.info('Enough free space ({}b), no cleanup'.format(free_bytes)) return "Pre-flight check done." -def arista_firmware_download(task, filename: str, httpd_url: str) -> None: +def arista_post_flight_check(task, post_waittime: int, job_id: Optional[str] = None) -> str: + """ + NorNir task to update device facts after a switch have been upgraded + + Args: + task: NorNir task + post_waittime: Time to wait before trying to gather facts + + Returns: + String, describing the result + + """ + set_thread_data(job_id) + logger = get_logger() + time.sleep(int(post_waittime)) + logger.info('Post-flight check wait ({}s) complete, starting check for {}'.format(post_waittime, task.host.name)) + with sqla_session() as session: + if Job.check_job_abort_status(session, job_id): + return "Post-flight aborted" + + try: + res = task.run(napalm_get, getters=["facts"]) + os_version = res[0].result['facts']['os_version'] + + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.hostname == task.host.name).one() + prev_os_version = dev.os_version + dev.os_version = os_version + if prev_os_version == os_version: + logger.error("OS version did not change, activation failed on {}".format(task.host.name)) + raise Exception("OS version did not change, activation failed") + else: + dev.confhash = None + dev.synchronized = False + except Exception as e: + logger.exception("Could not update OS version on device {}: {}".format(task.host.name, str(e))) + return 'Post-flight failed, could not update OS version: {}'.format(str(e)) + + return "Post-flight, OS version updated from {} to {}.".format(prev_os_version, + os_version) + + +def arista_firmware_download(task, filename: str, httpd_url: str, + job_id: Optional[str] = None) -> str: """ NorNir task to download firmware image from the HTTP server. @@ -61,12 +109,20 @@ def arista_firmware_download(task, filename: str, httpd_url: str) -> None: httpd_url: Base URL to the HTTP server Returns: - Nothing. + String, describing the result """ - logger.info('Downloading firmware for {}'.format(task.host.name)) + set_thread_data(job_id) + logger = get_logger() + with sqla_session() as session: + if Job.check_job_abort_status(session, job_id): + return "Firmware download aborted" url = httpd_url + '/' + filename + # Make sure netmiko doesn't use fast_cli because it will change delay_factor + # that is set in task.run below and cause early timeouts + net_connect = task.host.get_connection("netmiko", task.nornir.config) + net_connect.fast_cli = False try: with sqla_session() as session: @@ -79,25 +135,38 @@ def arista_firmware_download(task, filename: str, httpd_url: str) -> None: else: firmware_download_cmd = 'copy {} vrf MGMT flash:'.format(url) - res = task.run(netmiko_send_command, command_string='enable', - expect_string='.*#') - print_result(res) - res = task.run(netmiko_send_command, command_string=firmware_download_cmd.replace("//", "/"), + enable=True, delay_factor=30, - max_loops=200, - expect_string='.*Copy completed successfully.*') - print_result(res) + max_loops=200) + + if 'Copy completed successfully' in res.result: + return "Firmware download done." + else: + logger.debug("Firmware download failed on {} ('{}'): {}".format( + task.host.name, firmware_download_cmd, res.result)) + raise Exception("Copy command did not complete successfully: {}".format( + ', '.join(filter(lambda x: x.startswith('get:'), res.result.splitlines())) + )) + + except NornirSubTaskError as e: + subtask_result = e.result[0] + logger.error('{} failed to download firmware: {}'.format( + task.host.name, subtask_result)) + logger.debug('{} download subtask result: {}'.format( + task.host.name, subtask_result.result + )) + raise Exception('Failed to download firmware: {}'.format(subtask_result)) except Exception as e: - logger.info('{} failed to download firmware: {}'.format( + logger.error('{} failed to download firmware: {}'.format( task.host.name, e)) - raise Exception('Failed to download firmware') + raise Exception('Failed to download firmware: {}'.format(e)) return "Firmware download done." -def arista_firmware_activate(task, filename: str) -> None: +def arista_firmware_activate(task, filename: str, job_id: Optional[str] = None) -> str: """ NorNir task to modify the boot config for new firmwares. @@ -106,45 +175,53 @@ def arista_firmware_activate(task, filename: str) -> None: filename: Name of the new firmware image Returns: - Nope. + String, describing the result """ + set_thread_data(job_id) + logger = get_logger() + with sqla_session() as session: + if Job.check_job_abort_status(session, job_id): + return "Firmware activate aborted" + try: boot_file_cmd = 'boot system flash:{}'.format(filename) res = task.run(netmiko_send_command, command_string='enable', expect_string='.*#') - print_result(res) + + res = task.run(netmiko_send_command, + command_string='show boot-config | grep -o "\\w*{}\\w*"'.format(filename)) + if res.result == filename: + raise FirmwareAlreadyActiveException( + 'Firmware already activated in boot-config on {}'.format(task.host.name)) res = task.run(netmiko_send_command, command_string='conf t', expect_string='.*config.*#') - print_result(res) res = task.run(netmiko_send_command, command_string=boot_file_cmd) - print_result(res) res = task.run(netmiko_send_command, command_string='end', expect_string='.*#') - print_result(res) res = task.run(netmiko_send_command, command_string='show boot-config | grep -o "\\w*{}\\w*"'.format(filename)) - print_result(res) if not isinstance(res, MultiResult): raise Exception('Could not check boot-config on {}'.format(task.host.name)) if res.result != filename: raise Exception('Firmware not activated properly on {}'.format(task.host.name)) - + except FirmwareAlreadyActiveException as e: + raise e except Exception as e: - logger.exception('Failed to activate firmware: {}'.format(str(e))) + logger.exception('Failed to activate firmware on {}: {}'.format(task.host.name, str(e))) raise Exception('Failed to activate firmware') return "Firmware activate done." -def arista_device_reboot(task) -> None: +def arista_device_reboot(task, job_id: Optional[str] = None) -> str: """ NorNir task to reboot a single device. @@ -152,17 +229,21 @@ def arista_device_reboot(task) -> None: task: NorNir task. Returns: - Nothing. + String, describing the result """ + set_thread_data(job_id) + logger = get_logger() + with sqla_session() as session: + if Job.check_job_abort_status(session, job_id): + return "Reboot aborted" + try: res = task.run(netmiko_send_command, command_string='enable', expect_string='.*#') - print_result(res) res = task.run(netmiko_send_command, command_string='write', expect_string='.*#') - print_result(res) res = task.run(netmiko_send_command, command_string='reload force', max_loops=2, @@ -175,19 +256,24 @@ def arista_device_reboot(task) -> None: return "Device reboot done." -def device_upgrade_task(task, job_id: str, reboot: False, filename: str, +def device_upgrade_task(task, job_id: str, + filename: str, url: str, + reboot: Optional[bool] = False, download: Optional[bool] = False, pre_flight: Optional[bool] = False, + post_flight: Optional[bool] = False, + post_waittime: Optional[int] = 0, activate: Optional[bool] = False) -> NornirJobResult: # If pre-flight is selected, execute the pre-flight task which # will verify the amount of disk space and so on. + set_thread_data(job_id) + logger = get_logger() if pre_flight: logger.info('Running pre-flight check on {}'.format(task.host.name)) try: - res = task.run(task=arista_pre_flight_check) - print_result(res) + res = task.run(task=arista_pre_flight_check, job_id=job_id) except Exception as e: logger.exception("Exception while doing pre-flight check: {}". format(str(e))) @@ -196,7 +282,7 @@ def device_upgrade_task(task, job_id: str, reboot: False, filename: str, if res.failed: logger.exception('Pre-flight check failed for: {}'.format( ' '.join(res.failed_hosts.keys()))) - raise e + raise # If download is true, go ahead and download the firmware if download: @@ -205,8 +291,7 @@ def device_upgrade_task(task, job_id: str, reboot: False, filename: str, task.host.name)) try: res = task.run(task=arista_firmware_download, filename=filename, - httpd_url=url) - print_result(res) + httpd_url=url, job_id=job_id) except Exception as e: logger.exception('Exception while downloading firmware: {}'.format( str(e))) @@ -214,25 +299,58 @@ def device_upgrade_task(task, job_id: str, reboot: False, filename: str, # If download_only is false, continue to activate the newly downloaded # firmware and verify that it if present in the boot-config. + already_active = False if activate: logger.info('Activating firmware {} on {}'.format( filename, task.host.name)) try: - res = task.run(task=arista_firmware_activate, filename=filename) - print_result(res) + res = task.run(task=arista_firmware_activate, filename=filename, job_id=job_id) + except NornirSubTaskError as e: + subtask_result = e.result[0] + logger.debug('Exception while activating firmware for {}: {}'.format( + task.host.name, subtask_result)) + if subtask_result.exception: + if isinstance(subtask_result.exception, FirmwareAlreadyActiveException): + already_active = True + logger.info("Firmware already active, skipping reboot and post_flight: {}". + format(subtask_result.exception)) + else: + logger.exception('Firmware activate subtask exception for {}: {}'.format( + task.host.name, str(subtask_result.exception) + )) + raise e + else: + logger.error('Activate subtask result for {}: {}'.format( + task.host.name, subtask_result.result + )) + raise e except Exception as e: - logger.exception('Exception while activating firmware: {}'.format( - str(e))) + logger.exception('Exception while activating firmware for {}: {}'.format( + task.host.name, str(e))) raise e # Reboot the device if needed, we will then lose the connection. - if reboot: + if reboot and not already_active: logger.info('Rebooting {}'.format(task.host.name)) try: - res = task.run(task=arista_device_reboot) + res = task.run(task=arista_device_reboot, job_id=job_id) except Exception as e: pass + # If post-flight is selected, execute the post-flight task which + # will update device facts for the selected devices + if post_flight and not already_active: + logger.info('Running post-flight check on {}, delay start by {}s'.format( + task.host.name, post_waittime)) + try: + res = task.run(task=arista_post_flight_check, post_waittime=post_waittime, job_id=job_id) + except Exception as e: + logger.exception('Failed to run post-flight check: {}'.format(str(e))) + else: + if res.failed: + logger.error('Post-flight check failed for: {}'.format( + ' '.join(res.failed_hosts.keys()))) + if job_id: with redis_session() as db: db.lpush('finished_devices_' + str(job_id), task.host.name) @@ -247,9 +365,12 @@ def device_upgrade(download: Optional[bool] = False, url: Optional[str] = None, job_id: Optional[str] = None, pre_flight: Optional[bool] = False, + post_flight: Optional[bool] = False, + post_waittime: Optional[int] = 600, reboot: Optional[bool] = False, scheduled_by: Optional[str] = None) -> NornirJobResult: + logger = get_logger() nr = cnaas_init() if hostname: nr_filtered, dev_count, _ = inventory_selector(nr, hostname=hostname) @@ -262,6 +383,8 @@ def device_upgrade(download: Optional[bool] = False, logger.info("Device(s) selected for firmware upgrade ({}): {}".format( dev_count, ", ".join(device_list) )) + logger.info(f"Upgrade tasks selected: pre_flight = {pre_flight}, download = {download}, " + + f"activate = {activate}, reboot = {reboot}, post_flight = {post_flight}") # Make sure we only upgrade Arista access switches for device in device_list: @@ -276,14 +399,16 @@ def device_upgrade(download: Optional[bool] = False, # Start tasks to take care of the upgrade try: - nrresult = nr_filtered.run(task=device_upgrade_task, job_id=job_id, + nrresult = nr_filtered.run(task=device_upgrade_task, + job_id=job_id, download=download, filename=filename, url=url, pre_flight=pre_flight, + post_flight=post_flight, + post_waittime=post_waittime, reboot=reboot, activate=activate) - print_result(nrresult) except Exception as e: logger.exception('Exception while upgrading devices: {}'.format( str(e))) diff --git a/src/cnaas_nms/confpush/get.py b/src/cnaas_nms/confpush/get.py index b0aa4f18..e9c377f4 100644 --- a/src/cnaas_nms/confpush/get.py +++ b/src/cnaas_nms/confpush/get.py @@ -2,26 +2,23 @@ import re import hashlib -from typing import Optional, Tuple, List, Dict +from typing import Optional, List, Dict -from nornir.core.deserializer.inventory import Inventory from nornir.core.filter import F -from nornir.plugins.tasks import networking -from nornir.plugins.functions.text import print_result from nornir.core.task import AggregatedResult -from nornir.plugins.tasks.networking import napalm_get +from nornir_napalm.plugins.tasks import napalm_get +from nornir_utils.plugins.functions import print_result import cnaas_nms.confpush.nornir_helper from cnaas_nms.db.session import sqla_session from cnaas_nms.db.device import Device, DeviceType -from cnaas_nms.db.linknet import Linknet from cnaas_nms.tools.log import get_logger from cnaas_nms.db.interface import Interface, InterfaceConfigType def get_inventory(): nr = cnaas_nms.confpush.nornir_helper.cnaas_init() - return Inventory.serialize(nr.inventory).dict() + return nr.dict() def get_running_config(hostname): @@ -62,7 +59,7 @@ def get_facts(hostname: Optional[str] = None, group: Optional[str] = None)\ else: nr_filtered = nr - result = nr_filtered.run(task=networking.napalm_get, getters=["facts"]) + result = nr_filtered.run(task=napalm_get, getters=["facts"]) print_result(result) return result @@ -87,31 +84,59 @@ def get_neighbors(hostname: Optional[str] = None, group: Optional[str] = None)\ else: nr_filtered = nr - result = nr_filtered.run(task=networking.napalm_get, getters=["lldp_neighbors"]) + result = nr_filtered.run(task=napalm_get, getters=["lldp_neighbors"]) print_result(result) return result -def get_uplinks(session, hostname: str) -> Dict[str, str]: +def get_uplinks(session, hostname: str, recheck: bool = False, + neighbors: Optional[List[Device]] = None, + linknets = None) -> Dict[str, str]: """Returns dict with mapping of interface -> neighbor hostname""" logger = get_logger() - # TODO: check if uplinks are already saved in database? uplinks = {} - dev = session.query(Device).filter(Device.hostname == hostname).one() + dev: Device = session.query(Device).filter(Device.hostname == hostname).one() + if not recheck: + current_uplinks: List[Interface] = session.query(Interface).\ + filter(Interface.device == dev).\ + filter(Interface.configtype == InterfaceConfigType.ACCESS_UPLINK).all() + uplink_intf: Interface + for uplink_intf in current_uplinks: + try: + uplinks[uplink_intf.name] = uplink_intf.data['neighbor'] + except Exception as e: + continue + if len(uplinks) == 2: + logger.debug("Existing uplinks for device {} found: {}". + format(hostname, ', '.join(["{}: {}".format(ifname, hostname) + for ifname, hostname in uplinks.items()]))) + return uplinks + neighbor_d: Device - for neighbor_d in dev.get_neighbors(session): + if not neighbors: + neighbors = dev.get_neighbors(session) + + for neighbor_d in neighbors: if neighbor_d.device_type == DeviceType.DIST: local_if = dev.get_neighbor_local_ifname(session, neighbor_d) - # TODO: check that dist interface is configured as downlink + # Neighbor interface ifclass is already verified in + # update_linknets -> verify_peer_iftype if local_if: uplinks[local_if] = neighbor_d.hostname elif neighbor_d.device_type == DeviceType.ACCESS: - intfs: Interface = session.query(Interface).filter(Interface.device == neighbor_d). \ - filter(InterfaceConfigType == InterfaceConfigType.ACCESS_DOWNLINK).all() - local_if = dev.get_neighbor_local_ifname(session, neighbor_d) - remote_if = neighbor_d.get_neighbor_local_ifname(session, dev) + intfs: Interface = session.query(Interface).filter(Interface.device == neighbor_d).\ + filter(Interface.configtype == InterfaceConfigType.ACCESS_DOWNLINK).all() + if not intfs: + continue + try: + local_if = dev.get_neighbor_local_ifname(session, neighbor_d) + remote_if = neighbor_d.get_neighbor_local_ifname(session, dev) + except ValueError as e: + logger.debug("Ignoring possible uplinks to neighbor {}: {}".format( + neighbor_d.hostname, e)) + continue intf: Interface for intf in intfs: @@ -150,7 +175,7 @@ def get_interfaces(hostname: str) -> AggregatedResult: nr_filtered = nr.filter(name=hostname) if len(nr_filtered.inventory) != 1: raise ValueError(f"Hostname {hostname} not found in inventory") - nrresult = nr_filtered.run(task=networking.napalm_get, getters=["interfaces"]) + nrresult = nr_filtered.run(task=napalm_get, getters=["interfaces"]) return nrresult @@ -194,113 +219,52 @@ def get_interfacedb_ifs(session, hostname: str) -> List[str]: return ret -def update_inventory(hostname: str, site='default') -> dict: - """Update CMDB inventory with information gathered from device. - - Args: - hostname (str): Hostname of device to update - - Returns: - python dict with any differances of update - - Raises: - napalm.base.exceptions.ConnectionException: Can't connect to specified device - """ - # TODO: Handle napalm.base.exceptions.ConnectionException ? - result = get_facts(hostname=hostname)[hostname][0] - if result.failed: - raise Exception - facts = result.result['facts'] - with sqla_session() as session: - d = session.query(Device).\ - filter(Device.hostname == hostname).\ - one() - attr_map = { - # Map NAPALM getfacts name -> device.Device member name - 'vendor': 'vendor', - 'model': 'model', - 'os_version': 'os_version', - 'serial_number': 'serial', - } - diff = {} - # Update any attributes that has changed, save diff - for dict_key, obj_mem in attr_map.items(): - obj_data = d.__getattribute__(obj_mem) - if facts[dict_key] and obj_data != facts[dict_key]: - diff[obj_mem] = {'old': obj_data, - 'new': facts[dict_key] - } - d.__setattr__(obj_mem, facts[dict_key]) - d.last_seen = datetime.datetime.now() - session.commit() - return diff - - -def update_linknets(session, hostname): - """Update linknet data for specified device using LLDP neighbor data. - """ - logger = get_logger() - result = get_neighbors(hostname=hostname)[hostname][0] - if result.failed: - raise Exception - neighbors = result.result['lldp_neighbors'] - - ret = [] - - local_device_inst = session.query(Device).filter(Device.hostname == hostname).one() - logger.debug("Updating linknets for device {} ...".format(local_device_inst.id)) - - for local_if, data in neighbors.items(): - logger.debug(f"Local: {local_if}, remote: {data[0]['hostname']} {data[0]['port']}") - remote_device_inst = session.query(Device).\ - filter(Device.hostname == data[0]['hostname']).one_or_none() - if not remote_device_inst: - logger.info(f"Unknown connected device: {data[0]['hostname']}") - continue - logger.debug(f"Remote device found, device id: {remote_device_inst.id}") - - # Check if linknet object already exists in database - local_devid = local_device_inst.id - check_linknet = session.query(Linknet).\ - filter( - ((Linknet.device_a_id == local_devid) & (Linknet.device_a_port == local_if)) - | - ((Linknet.device_b_id == local_devid) & (Linknet.device_b_port == local_if)) - | - ((Linknet.device_a_id == remote_device_inst.id) & - (Linknet.device_a_port == data[0]['port'])) - | - ((Linknet.device_b_id == remote_device_inst.id) & - (Linknet.device_b_port == data[0]['port'])) - ).one_or_none() - if check_linknet: - logger.debug(f"Found entry: {check_linknet.id}") - if ( - ( check_linknet.device_a_id == local_devid - and check_linknet.device_a_port == local_if - and check_linknet.device_b_id == remote_device_inst.id - and check_linknet.device_b_port == data[0]['port'] - ) - or - ( check_linknet.device_a_id == local_devid - and check_linknet.device_a_port == local_if - and check_linknet.device_b_id == remote_device_inst.id - and check_linknet.device_b_port == data[0]['port'] - ) - ): - # All info is the same, no update required - continue - else: - # TODO: update instead of delete+new insert? - session.delete(check_linknet) - session.commit() - - new_link = Linknet() - new_link.device_a = local_device_inst - new_link.device_a_port = local_if - new_link.device_b = remote_device_inst - new_link.device_b_port = data[0]['port'] - session.add(new_link) - ret.append(new_link.as_dict()) - session.commit() - return ret +def verify_peer_iftype(local_hostname: str, local_devtype: DeviceType, + local_device_settings: dict, local_if: str, + remote_hostname: str, remote_devtype: DeviceType, + remote_device_settings: dict, remote_if: str): + # Make sure interface with peers are configured in settings for CORE and DIST devices + if remote_devtype in [DeviceType.DIST, DeviceType.CORE]: + match = False + for intf in remote_device_settings['interfaces']: + if intf['name'] == remote_if: + match = True + if not match: + raise ValueError("Peer device interface is not configured: " + "{} {}".format(remote_hostname, + remote_if)) + if local_devtype in [DeviceType.DIST, DeviceType.CORE]: + match = False + for intf in local_device_settings['interfaces']: + if intf['name'] == local_if: + match = True + if not match: + raise ValueError("Local device interface is not configured: " + "{} {}".format(local_hostname, + local_if)) + + # Make sure linknets between CORE/DIST devices are configured as fabric + if local_devtype in [DeviceType.DIST, DeviceType.CORE] and \ + remote_devtype in [DeviceType.DIST, DeviceType.CORE]: + for intf in local_device_settings['interfaces']: + if intf['name'] == local_if and intf['ifclass'] != 'fabric': + raise ValueError("Local device interface is not configured as fabric: " + "{} {} ifclass: {}".format(local_hostname, + intf['name'], + intf['ifclass'])) + for intf in remote_device_settings['interfaces']: + if intf['name'] == remote_if and intf['ifclass'] != 'fabric': + raise ValueError("Peer device interface is not configured as fabric: " + "{} {} ifclass: {}".format(remote_hostname, + intf['name'], + intf['ifclass'])) + + # Make sure that an access switch is connected to an interface + # configured as "downlink" on the remote end + if local_devtype == DeviceType.ACCESS and remote_devtype == DeviceType.DIST: + for intf in remote_device_settings['interfaces']: + if intf['name'] == remote_if and intf['ifclass'] != 'downlink': + raise ValueError("Peer device interface is not configured as downlink: " + "{} {} ifclass: {}".format(remote_hostname, + intf['name'], + intf['ifclass'])) diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py index fb8feda1..c1b495c1 100644 --- a/src/cnaas_nms/confpush/init_device.py +++ b/src/cnaas_nms/confpush/init_device.py @@ -1,42 +1,53 @@ from typing import Optional, List -from ipaddress import IPv4Interface +from ipaddress import IPv4Interface, IPv4Address -from nornir.plugins.tasks import networking, text -from nornir.plugins.functions.text import print_result -from nornir.core.inventory import ConnectionOptions -from napalm.base.exceptions import SessionLockedException +from nornir_napalm.plugins.tasks import napalm_configure, napalm_get +from nornir_jinja2.plugins.tasks import template_file +from nornir_utils.plugins.functions import print_result from apscheduler.job import Job import yaml import os import cnaas_nms.confpush.nornir_helper import cnaas_nms.confpush.get +import cnaas_nms.confpush.underlay import cnaas_nms.db.helper from cnaas_nms.db.session import sqla_session from cnaas_nms.db.device import Device, DeviceState, DeviceType, DeviceStateException from cnaas_nms.db.interface import Interface, InterfaceConfigType from cnaas_nms.scheduler.scheduler import Scheduler from cnaas_nms.scheduler.wrapper import job_wrapper -from cnaas_nms.confpush.nornir_helper import NornirJobResult -from cnaas_nms.confpush.update import update_interfacedb_worker -from cnaas_nms.confpush.sync_devices import get_mlag_vars +from cnaas_nms.confpush.nornir_helper import NornirJobResult, cnaas_jinja_env +from cnaas_nms.confpush.update import update_interfacedb_worker, update_linknets, set_facts +from cnaas_nms.confpush.sync_devices import populate_device_vars, confcheck_devices, \ + sync_devices from cnaas_nms.db.git import RepoStructureException -from cnaas_nms.db.settings import get_settings from cnaas_nms.plugins.pluginmanager import PluginManagerHandler from cnaas_nms.db.reservedip import ReservedIP from cnaas_nms.tools.log import get_logger from cnaas_nms.scheduler.thread_data import set_thread_data +from cnaas_nms.tools.pki import generate_device_cert +from cnaas_nms.confpush.cert import arista_copy_cert +from cnaas_nms.tools.get_apidata import get_apidata class ConnectionCheckError(Exception): pass +class InitVerificationError(Exception): + pass + + class InitError(Exception): pass -def push_base_management_access(task, device_variables, job_id): +class NeighborError(Exception): + pass + + +def push_base_management(task, device_variables: dict, devtype: DeviceType, job_id): set_thread_data(job_id) logger = get_logger() logger.debug("Push basetemplate for host: {}".format(task.host.name)) @@ -50,42 +61,53 @@ def push_base_management_access(task, device_variables, job_id): raise RepoStructureException("File {} not found in template repo".format(mapfile)) with open(mapfile, 'r') as f: mapping = yaml.safe_load(f) - template = mapping['ACCESS']['entrypoint'] - - settings, settings_origin = get_settings(task.host.name, DeviceType.ACCESS) - - # Add all environment variables starting with TEMPLATE_SECRET_ to - # the list of configuration variables. The idea is to store secret - # configuration outside of the templates repository. - template_secrets = {} - for env in os.environ: - if env.startswith('TEMPLATE_SECRET_'): - template_secrets[env] = os.environ[env] - - # Merge dicts, this will overwrite interface list from settings - template_vars = {**settings, **device_variables, **template_secrets} + template = mapping[devtype.name]['entrypoint'] - r = task.run(task=text.template_file, + # TODO: install device certificate, using new hostname and reserved IP. + # exception on fail if tls_verify!=False + try: + device_cert_res = task.run( + task=ztp_device_cert, + job_id=job_id, + new_hostname=task.host.name, + management_ip=device_variables['mgmt_ip'] + ) + # TODO: handle exception from ztp_device_cert -> arista_copy_cert + except Exception as e: + logger.exception(e) + else: + if device_cert_res.failed: + if device_cert_required(): + logger.error("Unable to install device certificate for {}, aborting".format( + device_variables['host'])) + raise Exception(device_cert_res[0].exception) + else: + logger.debug("Unable to install device certificate for {}".format( + device_variables['host'])) + + r = task.run(task=template_file, name="Generate initial device config", template=template, + jinja_env=cnaas_jinja_env, path=f"{local_repo_path}/{task.host.platform}", - **template_vars) + **device_variables) #TODO: Handle template not found, variables not defined task.host["config"] = r.result # Use extra low timeout for this since we expect to loose connectivity after changing IP - task.host.connection_options["napalm"] = ConnectionOptions(extras={"timeout": 30}) + connopts_napalm = task.host.connection_options["napalm"] + connopts_napalm.extras["timeout"] = 30 try: - task.run(task=networking.napalm_configure, + task.run(task=napalm_configure, name="Push base management config", replace=True, configuration=task.host["config"], dry_run=False ) except Exception: - task.run(task=networking.napalm_get, getters=["facts"]) + task.run(task=napalm_get, getters=["facts"]) if not task.results[-1].failed: raise InitError("Device {} did not commit new base management config".format( task.host.name @@ -93,6 +115,8 @@ def push_base_management_access(task, device_variables, job_id): def pre_init_checks(session, device_id) -> Device: + """Find device with device_id and check that it's ready for init, returns + Device object or raises exception""" # Check that we can find device and that it's in the correct state to start init dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() if not dev: @@ -104,7 +128,7 @@ def pre_init_checks(session, device_id) -> Device: nr = cnaas_nms.confpush.nornir_helper.cnaas_init() nr_old_filtered = nr.filter(name=old_hostname) try: - nrresult_old = nr_old_filtered.run(task=networking.napalm_get, getters=["facts"]) + nrresult_old = nr_old_filtered.run(task=napalm_get, getters=["facts"]) except Exception as e: raise ConnectionCheckError(f"Failed to connect to device_id {device_id}: {str(e)}") if nrresult_old.failed: @@ -113,6 +137,106 @@ def pre_init_checks(session, device_id) -> Device: return dev +def pre_init_check_neighbors(session, dev: Device, devtype: DeviceType, + linknets: List[dict], + expected_neighbors: Optional[List[str]] = None, + mlag_peer_dev: Optional[Device] = None) -> List[str]: + """Check for compatible neighbors + Args: + session: SQLAlchemy session + dev: Device object to check + devtype: The target device type (not the same as current during init) + linknets: List of linknets to check for compatible neighbors + expected_neighbors: Optional list to manually specify neighbors + Returns: + List of compatible neighbor hostnames + """ + logger = get_logger() + verified_neighbors = [] + if expected_neighbors is not None and len(expected_neighbors) == 0: + logger.debug("expected_neighbors explicitly set to empty list, skipping neighbor checks") + return [] + if not linknets: + raise Exception("No linknets were specified to check_neighbors") + + if devtype == DeviceType.ACCESS: + neighbors = [] + uplinks = [] + for linknet in linknets: + if linknet['device_a_hostname'] == linknet['device_b_hostname']: + continue # don't add loopback cables as neighbors + elif linknet['device_a_hostname'] == dev.hostname: + if mlag_peer_dev and linknet['device_b_hostname'] == mlag_peer_dev.hostname: + continue # only add mlag peer linknet in one direction to avoid duplicate + else: + neighbor = linknet['device_b_hostname'] + elif linknet['device_b_hostname'] == dev.hostname: + neighbor = linknet['device_a_hostname'] + elif mlag_peer_dev: + if linknet['device_a_hostname'] == mlag_peer_dev.hostname: + neighbor = linknet['device_b_hostname'] + elif linknet['device_b_hostname'] == mlag_peer_dev.hostname: + neighbor = linknet['device_a_hostname'] + else: + raise Exception("Own hostname not found in linknet") + neighbor_dev: Device = session.query(Device). \ + filter(Device.hostname == neighbor).one_or_none() + if not neighbor_dev: + raise Exception("Neighbor device {} not found in database".format(neighbor)) + if neighbor_dev.device_type in [DeviceType.ACCESS, DeviceType.DIST]: + uplinks.append(neighbor) + + neighbors.append(neighbor) + try: + cnaas_nms.db.helper.find_mgmtdomain(session, uplinks) + except Exception as e: + raise InitVerificationError(str(e)) + else: + verified_neighbors = neighbors + elif devtype in [DeviceType.CORE, DeviceType.DIST]: + for linknet in linknets: + if linknet['device_a_hostname'] == dev.hostname: + neighbor = linknet['device_b_hostname'] + elif linknet['device_b_hostname'] == dev.hostname: + neighbor = linknet['device_a_hostname'] + else: + raise Exception("Own hostname not found in linknet") + if expected_neighbors: + if neighbor in expected_neighbors: + verified_neighbors.append(neighbor) + # Neighbor was explicitly set -> skip verification of neighbor devtype + continue + + neighbor_dev: Device = session.query(Device).\ + filter(Device.hostname == neighbor).one_or_none() + if not neighbor_dev: + raise Exception("Neighbor device {} not found in database".format(neighbor)) + if devtype == DeviceType.CORE: + if neighbor_dev.device_type == DeviceType.DIST: + verified_neighbors.append(neighbor) + else: + logger.warn("Neighbor device {} is of unexpected device type {}, ignoring".format( + neighbor, neighbor_dev.device_type.name + )) + else: + if neighbor_dev.device_type == DeviceType.CORE: + verified_neighbors.append(neighbor) + else: + logger.warn("Neighbor device {} is of unexpected device type {}, ignoring".format( + neighbor, neighbor_dev.device_type.name + )) + + if expected_neighbors: + if len(expected_neighbors) != len(verified_neighbors): + raise InitVerificationError("Not all expected neighbors were detected") + else: + if len(verified_neighbors) < 2: + raise InitVerificationError("Not enough compatible neighbors ({} of 2) were detected".format( + len(verified_neighbors) + )) + return verified_neighbors + + def pre_init_check_mlag(session, dev, mlag_peer_dev): intfs: Interface = session.query(Interface).filter(Interface.device == dev).\ filter(InterfaceConfigType == InterfaceConfigType.MLAG_PEER).all() @@ -126,6 +250,43 @@ def pre_init_check_mlag(session, dev, mlag_peer_dev): )) +def ztp_device_cert(task, job_id: str, new_hostname: str, management_ip: str) -> str: + set_thread_data(job_id) + logger = get_logger() + + try: + ipv4: IPv4Address = IPv4Address(management_ip) + generate_device_cert(new_hostname, ipv4_address=ipv4) + except Exception as e: + raise Exception("Could not generate certificate for device {}: {}".format( + new_hostname, e + )) + + if task.host.platform == "eos": + try: + # TODO: subtaskerror? + res = task.run(task=arista_copy_cert, + job_id=job_id) + except Exception as e: + logger.exception('Exception while copying certificates: {}'.format( + str(e))) + raise e + else: + return "Install device certificate not supported on platform: {}".format( + task.host.platform + ) + return "Device certificate installed for {}".format(new_hostname) + + +def device_cert_required() -> bool: + apidata = get_apidata() + if 'verify_tls_device' in apidata and type(apidata['verify_tls_device']) == bool and \ + not apidata['verify_tls_device']: + return False + else: + return True + + @job_wrapper def init_access_device_step1(device_id: int, new_hostname: str, mlag_peer_id: Optional[int] = None, @@ -158,15 +319,16 @@ def init_access_device_step1(device_id: int, new_hostname: str, with sqla_session() as session: dev = pre_init_checks(session, device_id) - cnaas_nms.confpush.get.update_linknets(session, dev.hostname) # update linknets using LLDP data + # update linknets using LLDP data + update_linknets(session, dev.hostname, DeviceType.ACCESS) # If this is the first device in an MLAG pair if mlag_peer_id and mlag_peer_new_hostname: mlag_peer_dev = pre_init_checks(session, mlag_peer_id) - cnaas_nms.confpush.get.update_linknets(session, mlag_peer_dev.hostname) - update_interfacedb_worker(session, dev, replace=True, delete=False, + update_linknets(session, mlag_peer_dev.hostname, DeviceType.ACCESS) + update_interfacedb_worker(session, dev, replace=True, delete_all=False, mlag_peer_hostname=mlag_peer_dev.hostname) - update_interfacedb_worker(session, mlag_peer_dev, replace=True, delete=False, + update_interfacedb_worker(session, mlag_peer_dev, replace=True, delete_all=False, mlag_peer_hostname=dev.hostname) uplink_hostnames = dev.get_uplink_peer_hostnames(session) uplink_hostnames += mlag_peer_dev.get_uplink_peer_hostnames(session) @@ -180,7 +342,7 @@ def init_access_device_step1(device_id: int, new_hostname: str, raise ValueError("mlag_peer_id and mlag_peer_new_hostname must be specified together") # If this device is not part of an MLAG pair else: - update_interfacedb_worker(session, dev, replace=True, delete=False) + update_interfacedb_worker(session, dev, replace=True, delete_all=False) uplink_hostnames = dev.get_uplink_peer_hostnames(session) # TODO: check compatability, same dist pair and same ports on dists @@ -200,31 +362,19 @@ def init_access_device_step1(device_id: int, new_hostname: str, session.add(reserved_ip) # Populate variables for template rendering mgmt_gw_ipif = IPv4Interface(mgmtdomain.ipv4_gw) - device_variables = { + mgmt_variables = { 'mgmt_ipif': str(IPv4Interface('{}/{}'.format(mgmt_ip, mgmt_gw_ipif.network.prefixlen))), 'mgmt_ip': str(mgmt_ip), 'mgmt_prefixlen': int(mgmt_gw_ipif.network.prefixlen), - 'interfaces': [], 'mgmt_vlan_id': mgmtdomain.vlan, 'mgmt_gw': mgmt_gw_ipif.ip, - 'device_model': dev.model, - 'device_os_version': dev.os_version } - intfs = session.query(Interface).filter(Interface.device == dev).all() - intf: Interface - for intf in intfs: - intfdata = None - if intf.data: - intfdata = dict(intf.data) - device_variables['interfaces'].append({ - 'name': intf.name, - 'ifclass': intf.configtype.name, - 'data': intfdata - }) - mlag_vars = get_mlag_vars(session, dev) - device_variables = {**device_variables, **mlag_vars} + device_variables = populate_device_vars(session, dev, new_hostname, DeviceType.ACCESS) + device_variables = { + **device_variables, + **mgmt_variables + } # Update device state - dev = session.query(Device).filter(Device.id == device_id).one() dev.hostname = new_hostname session.commit() hostname = dev.hostname @@ -233,14 +383,16 @@ def init_access_device_step1(device_id: int, new_hostname: str, nr_filtered = nr.filter(name=hostname) # step2. push management config - nrresult = nr_filtered.run(task=push_base_management_access, + nrresult = nr_filtered.run(task=push_base_management, device_variables=device_variables, + devtype=DeviceType.ACCESS, job_id=job_id) with sqla_session() as session: dev = session.query(Device).filter(Device.id == device_id).one() dev.management_ip = device_variables['mgmt_ip'] dev.state = DeviceState.INIT + dev.device_type = DeviceType.ACCESS # Remove the reserved IP since it's now saved in the device database instead reserved_ip = session.query(ReservedIP).filter(ReservedIP.device == dev).one_or_none() if reserved_ip: @@ -257,10 +409,14 @@ def init_access_device_step1(device_id: int, new_hostname: str, logger.exception("Error while running plugin hooks for allocated_ipv4: ".format(str(e))) # step3. register apscheduler job that continues steps + if mlag_peer_id and mlag_peer_new_hostname: + step2_delay = 30+60+30 # account for delayed start of peer device plus mgmt timeout + else: + step2_delay = 30 scheduler = Scheduler() next_job_id = scheduler.add_onetime_job( - 'cnaas_nms.confpush.init_device:init_access_device_step2', - when=30, + 'cnaas_nms.confpush.init_device:init_device_step2', + when=step2_delay, scheduled_by=scheduled_by, kwargs={'device_id': device_id, 'iteration': 1}) @@ -289,13 +445,169 @@ def init_access_device_step1(device_id: int, new_hostname: str, ) -def schedule_init_access_device_step2(device_id: int, iteration: int, - scheduled_by: str) -> Optional[Job]: +def check_neighbor_sync(session, hostnames: List[str]): + for hostname in hostnames: + dev: Device = session.query(Device).filter(Device.hostname == hostname).one_or_none() + if not dev: + raise NeighborError("Neighbor device {} not found".format(hostname)) + if not dev.state == DeviceState.MANAGED: + raise NeighborError("Neighbor device {} not in state MANAGED".format(hostname)) + if not dev.synchronized: + raise NeighborError("Neighbor device {} not synchronized".format(hostname)) + confcheck_devices(hostnames) + + +@job_wrapper +def init_fabric_device_step1(device_id: int, new_hostname: str, device_type: str, + neighbors: Optional[List[str]] = [], + job_id: Optional[str] = None, + scheduled_by: Optional[str] = None) -> NornirJobResult: + """Initialize fabric (CORE/DIST) device for management by CNaaS-NMS. + + Args: + device_id: Device to select for initialization + new_hostname: Hostname to configure on this device + device_type: String representing DeviceType + neighbors: Optional list of hostnames of peer devices + job_id: job_id provided by scheduler when adding job + scheduled_by: Username from JWT. + + Returns: + Nornir result object + + Raises: + DeviceStateException + ValueError + """ + logger = get_logger() + if DeviceType.has_name(device_type): + devtype = DeviceType[device_type] + else: + raise ValueError("Invalid 'device_type' provided") + + if devtype not in [DeviceType.CORE, DeviceType.DIST]: + raise ValueError("Init fabric device requires device type DIST or CORE") + + with sqla_session() as session: + dev = pre_init_checks(session, device_id) + + # Test update of linknets using LLDP data + linknets = update_linknets( + session, dev.hostname, devtype, ztp_hostname=new_hostname, dry_run=True) + + try: + verified_neighbors = pre_init_check_neighbors( + session, dev, devtype, linknets, neighbors) + logger.debug("Found valid neighbors for INIT of {}: {}".format( + new_hostname, ", ".join(verified_neighbors) + )) + check_neighbor_sync(session, verified_neighbors) + except Exception as e: + raise e + else: + dev.state = DeviceState.INIT + dev.device_type = devtype + session.commit() + # If neighbor check works, commit new linknets + # This will also mark neighbors as unsynced + linknets = update_linknets( + session, dev.hostname, devtype, ztp_hostname=new_hostname, dry_run=False) + logger.debug("New linknets for INIT of {} created: {}".format( + new_hostname, linknets + )) + + # Select and reserve a new management and infra IP for the device + ReservedIP.clean_reservations(session, device=dev) + session.commit() + + mgmt_ip = cnaas_nms.confpush.underlay.find_free_mgmt_lo_ip(session) + infra_ip = cnaas_nms.confpush.underlay.find_free_infra_ip(session) + + reserved_ip = ReservedIP(device=dev, ip=mgmt_ip) + session.add(reserved_ip) + dev.infra_ip = infra_ip + session.commit() + + mgmt_variables = { + 'mgmt_ipif': str(IPv4Interface('{}/32'.format(mgmt_ip))), + 'mgmt_prefixlen': 32, + 'infra_ipif': str(IPv4Interface('{}/32'.format(infra_ip))), + 'infra_ip': str(infra_ip), + } + + device_variables = populate_device_vars(session, dev, new_hostname, devtype) + device_variables = { + **device_variables, + **mgmt_variables + } + # Update device state + dev.hostname = new_hostname + session.commit() + hostname = dev.hostname + + nr = cnaas_nms.confpush.nornir_helper.cnaas_init() + nr_filtered = nr.filter(name=hostname) + + # TODO: certicate + + # step2. push management config + nrresult = nr_filtered.run(task=push_base_management, + device_variables=device_variables, + devtype=devtype, + job_id=job_id) + + with sqla_session() as session: + dev = session.query(Device).filter(Device.id == device_id).one() + dev.management_ip = mgmt_ip + # Remove the reserved IP since it's now saved in the device database instead + reserved_ip = session.query(ReservedIP).filter(ReservedIP.device == dev).one_or_none() + if reserved_ip: + session.delete(reserved_ip) + + # Plugin hook, allocated IP + try: + pmh = PluginManagerHandler() + pmh.pm.hook.allocated_ipv4(vrf='mgmt', ipv4_address=str(mgmt_ip), + ipv4_network=None, + hostname=hostname + ) + except Exception as e: + logger.exception("Error while running plugin hooks for allocated_ipv4: ".format(str(e))) + + # step3. resync neighbors + scheduler = Scheduler() + sync_nei_job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.sync_devices:sync_devices', + when=1, + scheduled_by=scheduled_by, + kwargs={'hostnames': verified_neighbors, 'dry_run': False}) + logger.info(f"Scheduled job {sync_nei_job_id} to resynchronize neighbors") + + # step4. register apscheduler job that continues steps + scheduler = Scheduler() + next_job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.init_device:init_device_step2', + when=60, + scheduled_by=scheduled_by, + kwargs={'device_id': device_id, 'iteration': 1}) + + logger.info("Init step 2 for {} scheduled as job # {}".format( + new_hostname, next_job_id + )) + + return NornirJobResult( + nrresult=nrresult, + next_job_id=next_job_id + ) + + +def schedule_init_device_step2(device_id: int, iteration: int, + scheduled_by: str) -> Optional[int]: max_iterations = 2 if iteration > 0 and iteration < max_iterations: scheduler = Scheduler() next_job_id = scheduler.add_onetime_job( - 'cnaas_nms.confpush.init_device:init_access_device_step2', + 'cnaas_nms.confpush.init_device:init_device_step2', when=(30*iteration), scheduled_by=scheduled_by, kwargs={'device_id': device_id, 'iteration': iteration+1}) @@ -305,10 +617,10 @@ def schedule_init_access_device_step2(device_id: int, iteration: int, @job_wrapper -def init_access_device_step2(device_id: int, iteration: int = -1, - job_id: Optional[str] = None, - scheduled_by: Optional[str] = None) -> \ - NornirJobResult: +def init_device_step2(device_id: int, iteration: int = -1, + job_id: Optional[str] = None, + scheduled_by: Optional[str] = None) -> \ + NornirJobResult: logger = get_logger() # step4+ in apjob: if success, update management ip and device state, trigger external stuff? with sqla_session() as session: @@ -318,14 +630,14 @@ def init_access_device_step2(device_id: int, iteration: int = -1, format(device_id, dev.state.name)) raise DeviceStateException("Device must be in state INIT to continue init step 2") hostname = dev.hostname + devtype: DeviceType = dev.device_type nr = cnaas_nms.confpush.nornir_helper.cnaas_init() nr_filtered = nr.filter(name=hostname) - nrresult = nr_filtered.run(task=networking.napalm_get, getters=["facts"]) + nrresult = nr_filtered.run(task=napalm_get, getters=["facts"]) if nrresult.failed: - next_job_id = schedule_init_access_device_step2(device_id, iteration, - scheduled_by) + next_job_id = schedule_init_device_step2(device_id, iteration, scheduled_by) if next_job_id: return NornirJobResult( nrresult=nrresult, @@ -344,12 +656,8 @@ def init_access_device_step2(device_id: int, iteration: int = -1, with sqla_session() as session: dev: Device = session.query(Device).filter(Device.id == device_id).one() dev.state = DeviceState.MANAGED - dev.device_type = DeviceType.ACCESS dev.synchronized = False - dev.serial = facts['serial_number'] - dev.vendor = facts['vendor'] - dev.model = facts['model'] - dev.os_version = facts['os_version'] + set_facts(dev, facts) management_ip = dev.management_ip dev.dhcp_ip = None @@ -359,7 +667,7 @@ def init_access_device_step2(device_id: int, iteration: int = -1, pmh = PluginManagerHandler() pmh.pm.hook.new_managed_device( hostname=hostname, - device_type=DeviceType.ACCESS.name, + device_type=devtype.name, serial_number=facts['serial_number'], vendor=facts['vendor'], model=facts['model'], @@ -369,22 +677,20 @@ def init_access_device_step2(device_id: int, iteration: int = -1, except Exception as e: logger.exception("Error while running plugin hooks for new_managed_device: ".format(str(e))) - return NornirJobResult( - nrresult = nrresult - ) + return NornirJobResult(nrresult=nrresult) def schedule_discover_device(ztp_mac: str, dhcp_ip: str, iteration: int, scheduled_by: str) -> Optional[Job]: - max_iterations = 5 - if iteration > 0 and iteration < max_iterations: + max_iterations = 3 + if 0 < iteration <= max_iterations: scheduler = Scheduler() next_job_id = scheduler.add_onetime_job( 'cnaas_nms.confpush.init_device:discover_device', when=(60*iteration), scheduled_by=scheduled_by, kwargs={'ztp_mac': ztp_mac, 'dhcp_ip': dhcp_ip, - 'iteration': iteration+1}) + 'iteration': iteration}) return next_job_id else: return None @@ -396,15 +702,16 @@ def set_hostname_task(task, new_hostname: str): local_repo_path = repo_config['templates_local'] template_vars = {} # host is already set by nornir r = task.run( - task=text.template_file, + task=template_file, name="Generate hostname config", template="hostname.j2", + jinja_env=cnaas_jinja_env, path=f"{local_repo_path}/{task.host.platform}", **template_vars ) task.host["config"] = r.result task.run( - task=networking.napalm_configure, + task=napalm_configure, name="Configure hostname", replace=False, configuration=task.host["config"], @@ -413,7 +720,7 @@ def set_hostname_task(task, new_hostname: str): @job_wrapper -def discover_device(ztp_mac: str, dhcp_ip: str, iteration=-1, +def discover_device(ztp_mac: str, dhcp_ip: str, iteration: int, job_id: Optional[str] = None, scheduled_by: Optional[str] = None): logger = get_logger() @@ -432,33 +739,33 @@ def discover_device(ztp_mac: str, dhcp_ip: str, iteration=-1, nr = cnaas_nms.confpush.nornir_helper.cnaas_init() nr_filtered = nr.filter(name=hostname) - nrresult = nr_filtered.run(task=networking.napalm_get, getters=["facts"]) + nrresult = nr_filtered.run(task=napalm_get, getters=["facts"]) if nrresult.failed: logger.info("Could not contact device with ztp_mac {} (attempt {})".format( ztp_mac, iteration )) - next_job_id = schedule_discover_device(ztp_mac, dhcp_ip, iteration, + next_job_id = schedule_discover_device(ztp_mac, dhcp_ip, iteration+1, scheduled_by) if next_job_id: return NornirJobResult( - nrresult = nrresult, - next_job_id = next_job_id + nrresult=nrresult, + next_job_id=next_job_id ) else: - return NornirJobResult(nrresult = nrresult) + return NornirJobResult(nrresult=nrresult) try: facts = nrresult[hostname][0].result['facts'] with sqla_session() as session: dev: Device = session.query(Device).filter(Device.ztp_mac == ztp_mac).one() - dev.serial = facts['serial_number'] - dev.vendor = facts['vendor'] - dev.model = facts['model'] - dev.os_version = facts['os_version'] + dev.serial = facts['serial_number'][:64] + dev.vendor = facts['vendor'][:64] + dev.model = facts['model'][:64] + dev.os_version = facts['os_version'][:64] dev.state = DeviceState.DISCOVERED new_hostname = dev.hostname - logger.info(f"Device with ztp_mac {ztp_mac} successfully scanned, " + - "moving to DISCOVERED state") + logger.info(f"Device with ztp_mac {ztp_mac} successfully scanned" + + f"(attempt {iteration}), moving to DISCOVERED state") except Exception as e: logger.exception("Could not update device with ztp_mac {} with new facts: {}".format( ztp_mac, str(e) diff --git a/src/cnaas_nms/confpush/interface_state.py b/src/cnaas_nms/confpush/interface_state.py index 1dd8bcaa..e5a5ff4d 100644 --- a/src/cnaas_nms/confpush/interface_state.py +++ b/src/cnaas_nms/confpush/interface_state.py @@ -1,10 +1,10 @@ from typing import List import yaml -from nornir.plugins.tasks.networking import napalm_get, napalm_configure -from nornir.plugins.tasks.text import template_file +from nornir_napalm.plugins.tasks import napalm_configure, napalm_get +from nornir_jinja2.plugins.tasks import template_file -import cnaas_nms.confpush.nornir_helper +from cnaas_nms.confpush.nornir_helper import cnaas_init, cnaas_jinja_env from cnaas_nms.db.device import Device, DeviceState, DeviceType from cnaas_nms.db.interface import Interface, InterfaceConfigType from cnaas_nms.db.session import sqla_session @@ -14,7 +14,7 @@ def get_interface_states(hostname) -> dict: logger = get_logger() - nr = cnaas_nms.confpush.nornir_helper.cnaas_init() + nr = cnaas_init() nr_filtered = nr.filter(name=hostname).filter(managed=True) if len(nr_filtered.inventory) != 1: raise ValueError(f"Hostname {hostname} not found in inventory") @@ -67,6 +67,7 @@ def bounce_task(task, interfaces: List[str]): task=template_file, name="Generate port bounce down config", template="bounce-down.j2", + jinja_env=cnaas_jinja_env, path=f"{local_repo_path}/{task.host.platform}", **template_vars ) @@ -81,6 +82,7 @@ def bounce_task(task, interfaces: List[str]): task=template_file, name="Generate port bounce up config", template="bounce-up.j2", + jinja_env=cnaas_jinja_env, path=f"{local_repo_path}/{task.host.platform}", **template_vars ) @@ -98,7 +100,7 @@ def bounce_interfaces(hostname: str, interfaces: List[str]) -> bool: Returns false if config did not change, and raises Exception if an error was encountered.""" pre_bounce_check(hostname, interfaces) - nr = cnaas_nms.confpush.nornir_helper.cnaas_init() + nr = cnaas_init() nr_filtered = nr.filter(name=hostname).filter(managed=True) if len(nr_filtered.inventory) != 1: raise ValueError(f"Hostname {hostname} not found in inventory") diff --git a/src/cnaas_nms/confpush/nornir_helper.py b/src/cnaas_nms/confpush/nornir_helper.py index 1405f6f3..8d3926df 100644 --- a/src/cnaas_nms/confpush/nornir_helper.py +++ b/src/cnaas_nms/confpush/nornir_helper.py @@ -1,10 +1,15 @@ from dataclasses import dataclass -from typing import Optional, Tuple, List +from typing import Optional, Tuple, List, Union +import os from nornir import InitNornir from nornir.core import Nornir from nornir.core.task import AggregatedResult, MultiResult from nornir.core.filter import F +from nornir.core.plugins.inventory import InventoryPluginRegister +from jinja2 import Environment as JinjaEnvironment + +from cnaas_nms.confpush.nornir_plugins.cnaas_inventory import CnaasInventory from cnaas_nms.scheduler.jobresult import JobResult @@ -14,13 +19,25 @@ class NornirJobResult(JobResult): change_score: Optional[float] = None +cnaas_jinja_env = JinjaEnvironment( + trim_blocks=True, + lstrip_blocks=True, + keep_trailing_newline=True) + + def cnaas_init() -> Nornir: + InventoryPluginRegister.register("CnaasInventory", CnaasInventory) nr = InitNornir( - core={"num_workers": 50}, + runner={ + "plugin": "threaded", + "options": { + "num_workers": 50 + } + }, inventory={ - "plugin": "cnaas_nms.confpush.nornir_plugins.cnaas_inventory.CnaasInventory" + "plugin": "CnaasInventory" }, - logging={"file": "/tmp/nornir.log", "level": "debug"} + logging={"log_file": "/tmp/nornir-pid{}.log".format(os.getpid()), "level": "DEBUG"} ) return nr @@ -45,7 +62,7 @@ def nr_result_serialize(result: AggregatedResult): def inventory_selector(nr: Nornir, resync: bool = True, - hostname: Optional[str] = None, + hostname: Optional[Union[str, List[str]]] = None, device_type: Optional[str] = None, group: Optional[str] = None) -> Tuple[Nornir, int, List[str]]: """Return a filtered Nornir inventory with only the selected devices @@ -53,7 +70,7 @@ def inventory_selector(nr: Nornir, resync: bool = True, Args: nr: Nornir object resync: Set to false if you want to filter out devices that are synchronized - hostname: Select device by hostname (string) + hostname: Select device by hostname (string) or list of hostnames (list) device_type: Select device by device_type (string) group: Select device by group (string) @@ -63,7 +80,12 @@ def inventory_selector(nr: Nornir, resync: bool = True, """ skipped_devices = [] if hostname: - nr_filtered = nr.filter(name=hostname).filter(managed=True) + if isinstance(hostname, str): + nr_filtered = nr.filter(name=hostname).filter(managed=True) + elif isinstance(hostname, list): + nr_filtered = nr.filter(filter_func=lambda h: h.name in hostname).filter(managed=True) + else: + raise ValueError("Can't select hostname based on type {}".type(hostname)) elif device_type: nr_filtered = nr.filter(F(groups__contains='T_'+device_type)).filter(managed=True) elif group: diff --git a/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py b/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py index 33fd2d3b..37b2e88f 100644 --- a/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py +++ b/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py @@ -1,23 +1,44 @@ import os import ipaddress -from nornir.core.deserializer.inventory import Inventory +from nornir.core.inventory import ( + Inventory, + Group, + Groups, + Host, + Hosts, + Defaults, + ConnectionOptions, + ParentGroups, +) from cnaas_nms.db.device import Device, DeviceType, DeviceState from cnaas_nms.db.settings import get_groups +from cnaas_nms.tools.pki import ssl_context import cnaas_nms.db.session -class CnaasInventory(Inventory): - def _get_credentials(self, devicestate): +class CnaasInventory: + @staticmethod + def _get_credentials(devicestate): + if devicestate == 'UNKNOWN': + return None, None + elif devicestate in ['UNMANAGED', 'MANAGED_NOIF']: + env_var = 'MANAGED' + elif devicestate == 'PRE_CONFIGURED': + env_var = 'DHCP_BOOT' + else: + env_var = devicestate + try: - username = os.environ['USERNAME_' + devicestate] - password = os.environ['PASSWORD_' + devicestate] + username = os.environ['USERNAME_' + env_var] + password = os.environ['PASSWORD_' + env_var] except Exception: raise ValueError('Could not find credentials for state ' + devicestate) return username, password - def _get_management_ip(self, management_ip, dhcp_ip): + @staticmethod + def _get_management_ip(management_ip, dhcp_ip): if issubclass(management_ip.__class__, ipaddress.IPv4Address): return str(management_ip) elif issubclass(dhcp_ip.__class__, ipaddress.IPv4Address): @@ -25,66 +46,74 @@ def _get_management_ip(self, management_ip, dhcp_ip): else: return None - def __init__(self, **kwargs): - hosts = {} + def load(self) -> Inventory: + defaults = Defaults( + connection_options={ + "napalm": ConnectionOptions(extras={ + "optional_args": { + # args to eAPI HttpsEapiConnection for EOS + "enforce_verification": True, + "context": ssl_context + } + }) + } + ) + insecure_device_states = [ + DeviceState.INIT, + DeviceState.DHCP_BOOT, + DeviceState.PRE_CONFIGURED, + DeviceState.DISCOVERED + ] + insecure_connection_options = { + "napalm": ConnectionOptions(extras={ + "optional_args": {"enforce_verification": False} + }) + } + + groups = Groups() + for device_type in list(DeviceType.__members__): + group_name = 'T_'+device_type + groups[group_name] = Group(name=group_name, defaults=defaults) + for device_state in list(DeviceState.__members__): + username, password = self._get_credentials(device_state) + group_name = 'S_'+device_state + groups[group_name] = Group( + name=group_name, username=username, password=password, defaults=defaults) + for group_name in get_groups(): + groups[group_name] = Group(name=group_name, defaults=defaults) + + hosts = Hosts() with cnaas_nms.db.session.sqla_session() as session: instance: Device for instance in session.query(Device): - hosts[instance.hostname] = { - 'platform': instance.platform, - 'groups': [ - 'T_'+instance.device_type.name, - 'S_'+instance.state.name - ], - 'data': { - 'synchronized': instance.synchronized, - 'managed': (True if instance.state == DeviceState.MANAGED else False) - } - } - for group in get_groups(instance.hostname): - hosts[instance.hostname]['groups'].append(group) hostname = self._get_management_ip(instance.management_ip, instance.dhcp_ip) - if hostname: - hosts[instance.hostname]['hostname'] = hostname + port = None if instance.port and isinstance(instance.port, int): - hosts[instance.hostname]['port'] = instance.port - groups = { - 'global': { - 'data': { - 'k': 'v' - } - } - } - for device_type in list(DeviceType.__members__): - groups['T_'+device_type] = {} - for device_type in list(DeviceState.__members__): - groups['S_'+device_type] = {} - for group in get_groups(): - groups[group] = {} - - # Get credentials for device in state DHCP_BOOT - username, password = self._get_credentials('DHCP_BOOT') - groups['S_DHCP_BOOT']['username'] = username - groups['S_DHCP_BOOT']['password'] = password + port = instance.port + host_groups = [ + 'T_' + instance.device_type.name, + 'S_' + instance.state.name + ] + for member_group in get_groups(instance.hostname): + host_groups.append(member_group) - # Get credentials for device in state DISCOVERED - username, password = self._get_credentials('DISCOVERED') - groups['S_DISCOVERED']['username'] = username - groups['S_DISCOVERED']['password'] = password - - # Get credentials for device in state INIT - username, password = self._get_credentials('INIT') - groups['S_INIT']['username'] = username - groups['S_INIT']['password'] = password - - # Get credentials for device in state MANAGED - username, password = self._get_credentials('MANAGED') - groups['S_MANAGED']['username'] = username - groups['S_MANAGED']['password'] = password - groups['S_UNMANAGED']['username'] = username - groups['S_UNMANAGED']['password'] = password + if instance.state in insecure_device_states: + host_connection_options = insecure_connection_options + else: + host_connection_options = None + hosts[instance.hostname] = Host( + name=instance.hostname, + hostname=hostname, + platform=instance.platform, + groups=ParentGroups(groups[g] for g in host_groups), + port=port, + data={ + 'synchronized': instance.synchronized, + 'managed': (True if instance.state == DeviceState.MANAGED else False) + }, + connection_options=host_connection_options, + defaults=defaults + ) - defaults = {'data': {'k': 'v'}} - super().__init__(hosts=hosts, groups=groups, defaults=defaults, - **kwargs) + return Inventory(hosts=hosts, groups=groups, defaults=defaults) diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py index 9333b58c..e432b7ae 100644 --- a/src/cnaas_nms/confpush/sync_devices.py +++ b/src/cnaas_nms/confpush/sync_devices.py @@ -2,15 +2,15 @@ import yaml from typing import Optional, List from ipaddress import IPv4Interface, IPv4Address -from statistics import median from hashlib import sha256 -from nornir.plugins.tasks import networking, text -from nornir.plugins.functions.text import print_result from nornir.core.task import MultiResult +from nornir_napalm.plugins.tasks import napalm_configure, napalm_get +from nornir_jinja2.plugins.tasks import template_file +from nornir_utils.plugins.functions import print_result import cnaas_nms.db.helper -from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector +from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector, cnaas_jinja_env from cnaas_nms.db.session import sqla_session, redis_session from cnaas_nms.confpush.get import calc_config_hash from cnaas_nms.confpush.changescore import calculate_score @@ -25,7 +25,6 @@ from cnaas_nms.scheduler.thread_data import set_thread_data from cnaas_nms.scheduler.scheduler import Scheduler -from nornir.plugins.tasks.networking import napalm_get AUTOPUSH_MAX_SCORE = 10 @@ -38,7 +37,7 @@ def generate_asn(ipv4_address: IPv4Address) -> Optional[int]: return PRIVATE_ASN_START + (ipv4_address.packed[2]*256 + ipv4_address.packed[3]) -def get_evpn_spines(session, settings: dict): +def get_evpn_peers(session, settings: dict): logger = get_logger() device_hostnames = [] for entry in settings['evpn_peers']: @@ -51,6 +50,11 @@ def get_evpn_spines(session, settings: dict): dev = session.query(Device).filter(Device.hostname == hostname).one_or_none() if dev: ret.append(dev) + # If no evpn_peers were specified return a list of all CORE devices instead + if not ret: + core_devs = session.query(Device).filter(Device.device_type == DeviceType.CORE).all() + for dev in core_devs: + ret.append(dev) return ret @@ -98,43 +102,47 @@ def get_mlag_vars(session, dev: Device) -> dict: return ret -def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, - job_id: Optional[str] = None, - scheduled_by: Optional[str] = None): - """ - Nornir task to generate config and push to device +def populate_device_vars(session, dev: Device, + ztp_hostname: Optional[str] = None, + ztp_devtype: Optional[DeviceType] = None): + logger = get_logger() + device_variables = { + 'device_model': dev.model, + 'device_os_version': dev.os_version + } - Args: - task: nornir task, sent by nornir when doing .run() - dry_run: Don't commit config to device, just do compare/diff - generate_only: Only generate text config, don't try to commit or - even do dry_run compare to running config + if ztp_hostname: + hostname: str = ztp_hostname + else: + hostname: str = dev.hostname - Returns: + if ztp_devtype: + devtype: DeviceType = ztp_devtype + elif dev.device_type != DeviceType.UNKNOWN: + devtype: DeviceType = dev.device_type + else: + raise Exception("Can't populate device vars for device type UNKNOWN") - """ - set_thread_data(job_id) - logger = get_logger() - hostname = task.host.name - with sqla_session() as session: - dev: Device = session.query(Device).filter(Device.hostname == hostname).one() - mgmt_ip = dev.management_ip - infra_ip = dev.infra_ip + mgmt_ip = dev.management_ip + if not ztp_hostname: if not mgmt_ip: raise Exception("Could not find management IP for device {}".format(hostname)) - devtype: DeviceType = dev.device_type - if isinstance(dev.platform, str): - platform: str = dev.platform else: - raise ValueError("Unknown platform: {}".format(dev.platform)) - settings, settings_origin = get_settings(hostname, devtype) - device_variables = { - 'mgmt_ip': str(mgmt_ip), - 'device_model': dev.model, - 'device_os_version': dev.os_version - } + device_variables['mgmt_ip'] = str(mgmt_ip) + + if isinstance(dev.platform, str): + platform: str = dev.platform + else: + raise ValueError("Unknown platform: {}".format(dev.platform)) + + settings, settings_origin = get_settings(hostname, devtype, dev.model) - if devtype == DeviceType.ACCESS: + if devtype == DeviceType.ACCESS: + if ztp_hostname: + access_device_variables = { + 'interfaces': [] + } + else: mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain_by_ip(session, dev.management_ip) if not mgmtdomain: raise Exception( @@ -147,69 +155,131 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, 'mgmt_gw': str(mgmt_gw_ipif.ip), 'mgmt_ipif': str(IPv4Interface('{}/{}'.format(mgmt_ip, mgmt_gw_ipif.network.prefixlen))), + 'mgmt_ip': str(mgmt_ip), 'mgmt_prefixlen': int(mgmt_gw_ipif.network.prefixlen), 'interfaces': [] } - intfs = session.query(Interface).filter(Interface.device == dev).all() - intf: Interface - for intf in intfs: - untagged_vlan = None - tagged_vlan_list = [] - intfdata = None - if intf.data: - if 'untagged_vlan' in intf.data: - untagged_vlan = resolve_vlanid(intf.data['untagged_vlan'], - settings['vxlans']) - if 'tagged_vlan_list' in intf.data: - tagged_vlan_list = resolve_vlanid_list(intf.data['tagged_vlan_list'], - settings['vxlans']) - intfdata = dict(intf.data) - access_device_variables['interfaces'].append({ - 'name': intf.name, - 'ifclass': intf.configtype.name, - 'untagged_vlan': untagged_vlan, - 'tagged_vlan_list': tagged_vlan_list, - 'data': intfdata - }) - mlag_vars = get_mlag_vars(session, dev) - device_variables = {**access_device_variables, **device_variables, **mlag_vars} - elif devtype == DeviceType.DIST or devtype == DeviceType.CORE: - asn = generate_asn(infra_ip) - fabric_device_variables = { + + intfs = session.query(Interface).filter(Interface.device == dev).all() + intf: Interface + for intf in intfs: + untagged_vlan = None + tagged_vlan_list = [] + intfdata = None + try: + ifindexnum: int = Interface.interface_index_num(intf.name) + except ValueError as e: + ifindexnum: int = 0 + if intf.data: + if 'untagged_vlan' in intf.data: + untagged_vlan = resolve_vlanid(intf.data['untagged_vlan'], + settings['vxlans']) + if 'tagged_vlan_list' in intf.data: + tagged_vlan_list = resolve_vlanid_list(intf.data['tagged_vlan_list'], + settings['vxlans']) + intfdata = dict(intf.data) + access_device_variables['interfaces'].append({ + 'name': intf.name, + 'ifclass': intf.configtype.name, + 'untagged_vlan': untagged_vlan, + 'tagged_vlan_list': tagged_vlan_list, + 'data': intfdata, + 'indexnum': ifindexnum + }) + mlag_vars = get_mlag_vars(session, dev) + device_variables = {**device_variables, + **access_device_variables, + **mlag_vars} + elif devtype == DeviceType.DIST or devtype == DeviceType.CORE: + infra_ip = dev.infra_ip + asn = generate_asn(infra_ip) + fabric_device_variables = { + 'interfaces': [], + 'bgp_ipv4_peers': [], + 'bgp_evpn_peers': [], + 'mgmtdomains': [], + 'asn': asn + } + if mgmt_ip and infra_ip: + mgmt_device_variables = { 'mgmt_ipif': str(IPv4Interface('{}/32'.format(mgmt_ip))), 'mgmt_prefixlen': 32, 'infra_ipif': str(IPv4Interface('{}/32'.format(infra_ip))), 'infra_ip': str(infra_ip), - 'interfaces': [], - 'bgp_ipv4_peers': [], - 'bgp_evpn_peers': [], - 'mgmtdomains': [], - 'asn': asn } - ifname_peer_map = dev.get_linknet_localif_mapping(session) - if 'interfaces' in settings and settings['interfaces']: - for intf in settings['interfaces']: - try: - ifindexnum: int = Interface.interface_index_num(intf['name']) - except ValueError as e: - ifindexnum: int = 0 - if 'ifclass' in intf and intf['ifclass'] == 'downlink': - data = {} - if intf['name'] in ifname_peer_map: - data['description'] = ifname_peer_map[intf['name']] + fabric_device_variables = {**fabric_device_variables, **mgmt_device_variables} + # find fabric neighbors + fabric_interfaces = {} + for neighbor_d in dev.get_neighbors(session): + if neighbor_d.device_type == DeviceType.DIST or neighbor_d.device_type == DeviceType.CORE: + # TODO: support multiple links to the same neighbor? + local_if = dev.get_neighbor_local_ifname(session, neighbor_d) + local_ipif = dev.get_neighbor_local_ipif(session, neighbor_d) + neighbor_ip = dev.get_neighbor_ip(session, neighbor_d) + if local_if: + fabric_interfaces[local_if] = { + 'name': local_if, + 'ifclass': 'fabric', + 'ipv4if': local_ipif, + 'peer_hostname': neighbor_d.hostname, + 'peer_infra_lo': str(neighbor_d.infra_ip), + 'peer_ip': str(neighbor_ip), + 'peer_asn': generate_asn(neighbor_d.infra_ip) + } + fabric_device_variables['bgp_ipv4_peers'].append({ + 'peer_hostname': neighbor_d.hostname, + 'peer_infra_lo': str(neighbor_d.infra_ip), + 'peer_ip': str(neighbor_ip), + 'peer_asn': generate_asn(neighbor_d.infra_ip) + }) + ifname_peer_map = dev.get_linknet_localif_mapping(session) + if 'interfaces' in settings and settings['interfaces']: + for intf in settings['interfaces']: + try: + ifindexnum: int = Interface.interface_index_num(intf['name']) + except ValueError as e: + ifindexnum: int = 0 + if 'ifclass' not in intf: + continue + if intf['ifclass'] == 'downlink': + data = {} + if intf['name'] in ifname_peer_map: + data['description'] = ifname_peer_map[intf['name']] + fabric_device_variables['interfaces'].append({ + 'name': intf['name'], + 'ifclass': intf['ifclass'], + 'indexnum': ifindexnum, + 'data': data + }) + elif intf['ifclass'] == 'custom': + fabric_device_variables['interfaces'].append({ + 'name': intf['name'], + 'ifclass': intf['ifclass'], + 'config': intf['config'], + 'indexnum': ifindexnum + }) + elif intf['ifclass'] == 'fabric': + if intf['name'] in fabric_interfaces: + fabric_device_variables['interfaces'].append( + {**fabric_interfaces[intf['name']], **{'indexnum': ifindexnum}} + ) + del fabric_interfaces[intf['name']] + else: fabric_device_variables['interfaces'].append({ 'name': intf['name'], 'ifclass': intf['ifclass'], 'indexnum': ifindexnum, - 'data': data - }) - elif 'ifclass' in intf and intf['ifclass'] == 'custom': - fabric_device_variables['interfaces'].append({ - 'name': intf['name'], - 'ifclass': intf['ifclass'], - 'config': intf['config'], - 'indexnum': ifindexnum + 'ipv4if': None, + 'peer_hostname': 'ztp', + 'peer_infra_lo': None, + 'peer_ip': None, + 'peer_asn': None }) + for local_if, data in fabric_interfaces.items(): + logger.warn(f"Interface {local_if} on device {hostname} not " + "configured as linknet because of wrong ifclass") + + if not ztp_hostname: for mgmtdom in cnaas_nms.db.helper.get_all_mgmtdomains(session, hostname): fabric_device_variables['mgmtdomains'].append({ 'id': mgmtdom.id, @@ -218,40 +288,17 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, 'description': mgmtdom.description, 'esi_mac': mgmtdom.esi_mac }) - # find fabric neighbors - fabric_links = [] - for neighbor_d in dev.get_neighbors(session): - if neighbor_d.device_type == DeviceType.DIST or neighbor_d.device_type == DeviceType.CORE: - # TODO: support multiple links to the same neighbor? - local_if = dev.get_neighbor_local_ifname(session, neighbor_d) - local_ipif = dev.get_neighbor_local_ipif(session, neighbor_d) - neighbor_ip = dev.get_neighbor_ip(session, neighbor_d) - if local_if: - fabric_device_variables['interfaces'].append({ - 'name': local_if, - 'ifclass': 'fabric', - 'ipv4if': local_ipif, - 'peer_hostname': neighbor_d.hostname, - 'peer_infra_lo': str(neighbor_d.infra_ip), - 'peer_ip': str(neighbor_ip), - 'peer_asn': generate_asn(neighbor_d.infra_ip) - }) - fabric_device_variables['bgp_ipv4_peers'].append({ - 'peer_hostname': neighbor_d.hostname, - 'peer_infra_lo': str(neighbor_d.infra_ip), - 'peer_ip': str(neighbor_ip), - 'peer_asn': generate_asn(neighbor_d.infra_ip) - }) - # populate evpn peers data - for neighbor_d in get_evpn_spines(session, settings): - if neighbor_d.hostname == dev.hostname: - continue - fabric_device_variables['bgp_evpn_peers'].append({ - 'peer_hostname': neighbor_d.hostname, - 'peer_infra_lo': str(neighbor_d.infra_ip), - 'peer_asn': generate_asn(neighbor_d.infra_ip) - }) - device_variables = {**fabric_device_variables, **device_variables} + # populate evpn peers data + for neighbor_d in get_evpn_peers(session, settings): + if neighbor_d.hostname == dev.hostname: + continue + fabric_device_variables['bgp_evpn_peers'].append({ + 'peer_hostname': neighbor_d.hostname, + 'peer_infra_lo': str(neighbor_d.infra_ip), + 'peer_asn': generate_asn(neighbor_d.infra_ip) + }) + device_variables = {**device_variables, + **fabric_device_variables} # Add all environment variables starting with TEMPLATE_SECRET_ to # the list of configuration variables. The idea is to store secret @@ -260,12 +307,39 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, for env in os.environ: if env.startswith('TEMPLATE_SECRET_'): template_secrets[env] = os.environ[env] - - # Merge device variables with settings before sending to template rendering + # Merge all dicts with variables into one, later row overrides # Device variables override any names from settings, for example the # interfaces list from settings are replaced with an interface list from # device variables that contains more information - template_vars = {**settings, **device_variables, **template_secrets} + device_variables = {**settings, + **device_variables, + **template_secrets} + return device_variables + + +def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, + job_id: Optional[str] = None, + scheduled_by: Optional[str] = None): + """ + Nornir task to generate config and push to device + + Args: + task: nornir task, sent by nornir when doing .run() + dry_run: Don't commit config to device, just do compare/diff + generate_only: Only generate text config, don't try to commit or + even do dry_run compare to running config + + Returns: + + """ + set_thread_data(job_id) + logger = get_logger() + hostname = task.host.name + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.hostname == hostname).one() + template_vars = populate_device_vars(session, dev) + platform = dev.platform + devtype = dev.device_type with open('/etc/cnaas-nms/repository.yml', 'r') as db_file: repo_config = yaml.safe_load(db_file) @@ -279,9 +353,10 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, template = mapping[devtype.name]['entrypoint'] logger.debug("Generate config for host: {}".format(task.host.name)) - r = task.run(task=text.template_file, + r = task.run(task=template_file, name="Generate device config", template=template, + jinja_env=cnaas_jinja_env, path=f"{local_repo_path}/{task.host.platform}", **template_vars) @@ -298,7 +373,7 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, task.host.name, task.host.hostname, task.host.port)) task.host.open_connection("napalm", configuration=task.nornir.config) - task.run(task=networking.napalm_configure, + task.run(task=napalm_configure, name="Sync device config", replace=True, configuration=task.host["config"], @@ -403,8 +478,24 @@ def update_config_hash(task): logger.debug("Config hash for {} updated to {}".format(task.host.name, new_config_hash)) +def confcheck_devices(hostnames: List[str], job_id=None): + nr = cnaas_init() + nr_filtered, dev_count, skipped_hostnames = \ + inventory_selector(nr, hostname=hostnames) + + try: + nrresult = nr_filtered.run(task=sync_check_hash, + job_id=job_id) + except Exception as e: + raise e + else: + if nrresult.failed: + raise Exception('Configuration hash check failed for {}'.format( + ' '.join(nrresult.failed_hosts.keys()))) + + @job_wrapper -def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = None, +def sync_devices(hostnames: Optional[List[str]] = None, device_type: Optional[str] = None, group: Optional[str] = None, dry_run: bool = True, force: bool = False, auto_push: bool = False, job_id: Optional[int] = None, scheduled_by: Optional[str] = None, resync: bool = False) -> NornirJobResult: @@ -432,9 +523,9 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No nr = cnaas_init() dev_count = 0 skipped_hostnames = [] - if hostname: + if hostnames: nr_filtered, dev_count, skipped_hostnames = \ - inventory_selector(nr, hostname=hostname) + inventory_selector(nr, hostname=hostnames) else: if device_type: nr_filtered, dev_count, skipped_hostnames = \ @@ -520,17 +611,29 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No logger.debug("Empty diff for host {}, 0 change score".format( host)) - if not dry_run: + nr_confighash = None + if dry_run and force: + # update config hash for devices that had an empty diff because local + # changes on a device can cause reordering of CLI commands that results + # in config hash mismatch even if the calculated diff was empty + def include_filter(host, include_list=unchanged_hosts): + if host.name in include_list: + return True + else: + return False + nr_confighash = nr_filtered.filter(filter_func=include_filter) + elif not dry_run: + # set new config hash for devices that was successfully updated def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts): if host.name in exclude_list: return False else: return True + nr_confighash = nr_filtered.filter(filter_func=exclude_filter) - # set new config hash for devices that was successfully updated - nr_successful = nr_filtered.filter(filter_func=exclude_filter) + if nr_confighash: try: - nrresult_confighash = nr_successful.run(task=update_config_hash) + nrresult_confighash = nr_confighash.run(task=update_config_hash) except Exception as e: logger.exception("Exception while updating config hashes: {}".format(str(e))) else: @@ -566,7 +669,7 @@ def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts): format(total_change_score, dry_run, len(device_list), len(changed_hosts))) next_job_id = None - if auto_push and len(device_list) == 1 and hostname and dry_run: + if auto_push and len(device_list) == 1 and hostnames and dry_run: if not changed_hosts: logger.info("None of the selected host has any changes (diff), skipping auto-push") elif total_change_score < AUTOPUSH_MAX_SCORE: @@ -575,11 +678,11 @@ def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts): 'cnaas_nms.confpush.sync_devices:sync_devices', when=0, scheduled_by=scheduled_by, - kwargs={'hostname': hostname, 'dry_run': False, 'force': force}) + kwargs={'hostnames': hostnames, 'dry_run': False, 'force': force}) logger.info(f"Auto-push scheduled live-run of commit as job id {next_job_id}") else: logger.info( - f"Auto-push of config to device {hostname} failed because change score of " + f"Auto-push of config to device {hostnames} failed because change score of " f"{total_change_score} is higher than auto-push limit {AUTOPUSH_MAX_SCORE}" ) @@ -605,7 +708,7 @@ def push_static_config(task, config: str, dry_run: bool = True, logger.debug("Push static config to device: {}".format(task.host.name)) - task.run(task=networking.napalm_configure, + task.run(task=napalm_configure, name="Push static config", replace=True, configuration=config, diff --git a/src/cnaas_nms/confpush/tests/data/testdata.yml b/src/cnaas_nms/confpush/tests/data/testdata.yml index 6ae88d4a..3aad5f2f 100644 --- a/src/cnaas_nms/confpush/tests/data/testdata.yml +++ b/src/cnaas_nms/confpush/tests/data/testdata.yml @@ -2,3 +2,4 @@ init_access_device_id: 13 init_access_old_hostname: mac-0800275C091F init_access_new_hostname: eosaccess update_hostname: mac-0800275C091F +copycert_hostname: eosdist1 diff --git a/src/cnaas_nms/confpush/tests/test_cert.py b/src/cnaas_nms/confpush/tests/test_cert.py new file mode 100644 index 00000000..2f5bdd6e --- /dev/null +++ b/src/cnaas_nms/confpush/tests/test_cert.py @@ -0,0 +1,31 @@ +import unittest +import pkg_resources +import yaml +import os + +from nornir_utils.plugins.functions import print_result + +from cnaas_nms.confpush.nornir_helper import cnaas_init +from cnaas_nms.confpush.cert import arista_copy_cert + + +class CertTests(unittest.TestCase): + def setUp(self): + data_dir = pkg_resources.resource_filename(__name__, 'data') + with open(os.path.join(data_dir, 'testdata.yml'), 'r') as f_testdata: + self.testdata = yaml.safe_load(f_testdata) + + def copy_cert(self): + nr = cnaas_init() + nr_filtered = nr.filter(name=self.testdata['copycert_hostname']) + + nrresult = nr_filtered.run( + task=arista_copy_cert + ) + if nrresult.failed: + print_result(nrresult) + self.assertFalse(nrresult.failed, "Task arista_copy_cert returned failed status") + + +if __name__ == '__main__': + unittest.main() diff --git a/src/cnaas_nms/confpush/tests/test_get.py b/src/cnaas_nms/confpush/tests/test_get.py index 79851bd6..0d656889 100644 --- a/src/cnaas_nms/confpush/tests/test_get.py +++ b/src/cnaas_nms/confpush/tests/test_get.py @@ -6,7 +6,10 @@ import yaml import os +import cnaas_nms.confpush.update from cnaas_nms.db.session import sqla_session +from cnaas_nms.db.device import DeviceType + class GetTests(unittest.TestCase): def setUp(self): @@ -30,13 +33,10 @@ def test_get_facts(self): result = cnaas_nms.confpush.get.get_facts(group='S_DHCP_BOOT') pprint.pprint(result) - def test_update_inventory(self): - diff = cnaas_nms.confpush.get.update_inventory(self.testdata['update_hostname']) - pprint.pprint(diff) - def test_update_links(self): with sqla_session() as session: - new_links = cnaas_nms.confpush.get.update_linknets(session, self.testdata['update_hostname']) + new_links = cnaas_nms.confpush.update.update_linknets( + session, self.testdata['update_hostname'], DeviceType.ACCESS) pprint.pprint(new_links) diff --git a/src/cnaas_nms/confpush/tests/test_init.py b/src/cnaas_nms/confpush/tests/test_init.py index 16141391..d67ba951 100644 --- a/src/cnaas_nms/confpush/tests/test_init.py +++ b/src/cnaas_nms/confpush/tests/test_init.py @@ -1,12 +1,11 @@ -import pprint import unittest import pkg_resources import yaml import os import time -from nornir.plugins.tasks import networking -from nornir.plugins.functions.text import print_result +from nornir_napalm.plugins.tasks import napalm_configure +from nornir_utils.plugins.functions import print_result from nornir.core.inventory import ConnectionOptions import cnaas_nms.confpush.init_device @@ -69,7 +68,7 @@ def reset_access_device(self): config = f_reset_config.read() print(config) nrresult = nr_filtered.run( - task=networking.napalm_configure, + task=napalm_configure, name="Reset config", replace=False, configuration=config, diff --git a/src/cnaas_nms/confpush/underlay.py b/src/cnaas_nms/confpush/underlay.py index 10bced9d..e83c2b81 100644 --- a/src/cnaas_nms/confpush/underlay.py +++ b/src/cnaas_nms/confpush/underlay.py @@ -6,6 +6,7 @@ from cnaas_nms.db.device import Device, DeviceType from cnaas_nms.db.linknet import Linknet from cnaas_nms.db.settings import get_settings +from cnaas_nms.db.reservedip import ReservedIP def find_free_infra_ip(session) -> Optional[IPv4Address]: @@ -29,10 +30,14 @@ def find_free_infra_ip(session) -> Optional[IPv4Address]: def find_free_mgmt_lo_ip(session) -> Optional[IPv4Address]: """Returns first free IPv4 infra IP.""" used_ips = [] + reserved_ips = [] device_query = session.query(Device). \ filter(Device.management_ip != None).options(load_only("management_ip")) for device in device_query: used_ips.append(device.management_ip) + reserved_ip_query = session.query(ReservedIP).options(load_only("ip")) + for reserved_ip in reserved_ip_query: + reserved_ips.append(reserved_ip.ip) settings, settings_origin = get_settings(device_type=DeviceType.CORE) mgmt_lo_net = IPv4Network(settings['underlay']['mgmt_lo_net']) @@ -40,6 +45,8 @@ def find_free_mgmt_lo_ip(session) -> Optional[IPv4Address]: ipaddr = IPv4Address(net.network_address) if ipaddr in used_ips: continue + if ipaddr in reserved_ips: + continue else: return ipaddr return None diff --git a/src/cnaas_nms/confpush/update.py b/src/cnaas_nms/confpush/update.py index 965c101b..57831edb 100644 --- a/src/cnaas_nms/confpush/update.py +++ b/src/cnaas_nms/confpush/update.py @@ -1,47 +1,67 @@ from typing import Optional, List -from nornir.plugins.tasks import networking +from nornir_napalm.plugins.tasks import napalm_get +from cnaas_nms.confpush.underlay import find_free_infra_linknet +from cnaas_nms.db.linknet import Linknet from cnaas_nms.db.session import sqla_session from cnaas_nms.db.device import Device, DeviceType, DeviceState from cnaas_nms.db.interface import Interface, InterfaceConfigType from cnaas_nms.confpush.get import get_interfaces_names, get_uplinks, \ - filter_interfaces, get_mlag_ifs + filter_interfaces, get_mlag_ifs, get_neighbors, verify_peer_iftype +from cnaas_nms.db.settings import get_settings from cnaas_nms.tools.log import get_logger from cnaas_nms.scheduler.wrapper import job_wrapper from cnaas_nms.confpush.nornir_helper import NornirJobResult +from cnaas_nms.scheduler.jobresult import DictJobResult import cnaas_nms.confpush.nornir_helper -def update_interfacedb_worker(session, dev: Device, replace: bool, delete: bool, +def update_interfacedb_worker(session, dev: Device, replace: bool, delete_all: bool, mlag_peer_hostname: Optional[str] = None) -> List[dict]: - """Perform actual work of updating database for update_interfacedb""" + """Perform actual work of updating database for update_interfacedb. + If replace is set to true, configtype and data will get overwritten. + If delete_all is set to true, delete all interfaces from database. + Return list of new/updated interfaces, or empty if delete_all was set.""" logger = get_logger() ret = [] - iflist = get_interfaces_names(dev.hostname) - uplinks = get_uplinks(session, dev.hostname) + current_iflist = session.query(Interface).filter(Interface.device == dev).all() + unmatched_iflist = [] + current_intf: Interface + for current_intf in current_iflist: + if delete_all: + logger.debug("Deleting interface {} on device {} from interface DB".format( + current_intf.name, dev.hostname + )) + session.delete(current_intf) + else: + unmatched_iflist.append(current_intf) + if delete_all: + session.commit() + return ret + + iflist = get_interfaces_names(dev.hostname) # query nornir for current interfaces + uplinks = get_uplinks(session, dev.hostname, recheck=replace) if mlag_peer_hostname: mlag_ifs = get_mlag_ifs(session, dev.hostname, mlag_peer_hostname) else: mlag_ifs = {} phy_interfaces = filter_interfaces(iflist, platform=dev.platform, include='physical') + if not phy_interfaces: + raise Exception("Could not find any physical interfaces for device {}".format(dev.hostname)) for intf_name in phy_interfaces: intf: Interface = session.query(Interface).filter(Interface.device == dev). \ filter(Interface.name == intf_name).one_or_none() + if intf in unmatched_iflist: + unmatched_iflist.remove(intf) if intf: new_intf = False else: new_intf = True intf: Interface = Interface() - if not new_intf and delete: # 'not new_intf' means interface exists in database - logger.debug("Deleting interface {} on device {} from interface DB".format( - intf_name, dev.hostname - )) - session.delete(intf) - continue - elif not new_intf and not replace: + if not new_intf and not replace: continue logger.debug("New/updated physical interface found on device {}: {}".format( dev.hostname, intf_name @@ -59,15 +79,32 @@ def update_interfacedb_worker(session, dev: Device, replace: bool, delete: bool, if new_intf: session.add(intf) ret.append(intf.as_dict()) + + # Remove interfaces that no longer exist on device + for unmatched_intf in unmatched_iflist: + protected_interfaces = [InterfaceConfigType.ACCESS_UPLINK, + InterfaceConfigType.MLAG_PEER] + if unmatched_intf.configtype in protected_interfaces: + logger.warn("Interface of protected type disappeared from {} ignoring: {}".format( + dev.hostname, unmatched_intf.name + )) + else: + logger.info("Deleting interface {} from {} because it disappeared on device".format( + unmatched_intf.name, dev.hostname + )) + session.delete(unmatched_intf) session.commit() return ret -def update_interfacedb(hostname: str, replace: bool = False, delete: bool = False) \ - -> List[dict]: +@job_wrapper +def update_interfacedb(hostname: str, replace: bool = False, delete_all: bool = False, + mlag_peer_hostname: Optional[str] = None, + job_id: Optional[str] = None, + scheduled_by: Optional[str] = None) -> DictJobResult: """Update interface DB with any new physical interfaces for specified device. If replace is set, any existing records in the database will get overwritten. - If delete is set, all entries in database for this device will be removed. + If delete_all is set, all entries in database for this device will be removed. Returns: List of interfaces that was added to DB @@ -81,11 +118,11 @@ def update_interfacedb(hostname: str, replace: bool = False, delete: bool = Fals if dev.device_type != DeviceType.ACCESS: raise ValueError("This function currently only supports access devices") - result = update_interfacedb_worker(session, dev, replace, delete) + result = update_interfacedb_worker(session, dev, replace, delete_all, mlag_peer_hostname) if result: dev.synchronized = False - return result + return DictJobResult(result={"interfaces": result}) def reset_interfacedb(hostname: str): @@ -98,6 +135,29 @@ def reset_interfacedb(hostname: str): return ret +def set_facts(dev: Device, facts: dict) -> dict: + attr_map = { + # Map NAPALM getfacts name -> device.Device member name + 'vendor': 'vendor', + 'model': 'model', + 'os_version': 'os_version', + 'serial_number': 'serial', + } + diff = {} + # Update any attributes that has changed + for dict_key, obj_member in attr_map.items(): + obj_data = dev.__getattribute__(obj_member) + maxlen = Device.__dict__[obj_member].property.columns[0].type.length + fact_data = facts[dict_key][:maxlen] + if fact_data and obj_data != fact_data: + diff[obj_member] = { + 'old': obj_data, + 'new': fact_data + } + dev.__setattr__(obj_member, fact_data) + return diff + + @job_wrapper def update_facts(hostname: str, job_id: Optional[str] = None, @@ -116,7 +176,7 @@ def update_facts(hostname: str, nr = cnaas_nms.confpush.nornir_helper.cnaas_init() nr_filtered = nr.filter(name=hostname) - nrresult = nr_filtered.run(task=networking.napalm_get, getters=["facts"]) + nrresult = nr_filtered.run(task=napalm_get, getters=["facts"]) if nrresult.failed: logger.error("Could not contact device with hostname {}".format(hostname)) @@ -125,11 +185,9 @@ def update_facts(hostname: str, facts = nrresult[hostname][0].result['facts'] with sqla_session() as session: dev: Device = session.query(Device).filter(Device.hostname == hostname).one() - dev.serial = facts['serial_number'] - dev.vendor = facts['vendor'] - dev.model = facts['model'] - dev.os_version = facts['os_version'] - logger.debug("Updating facts for device {}: {}, {}, {}, {}".format( + diff = set_facts(dev, facts) + + logger.debug("Updating facts for device {}, new values: {}, {}, {}, {}".format( hostname, facts['serial_number'], facts['vendor'], facts['model'], facts['os_version'] )) except Exception as e: @@ -139,4 +197,132 @@ def update_facts(hostname: str, logger.debug("Get facts nrresult for hostname {}: {}".format(hostname, nrresult)) raise e - return NornirJobResult(nrresult=nrresult) + return DictJobResult(result={"diff": diff}) + + +def update_linknets(session, hostname: str, devtype: DeviceType, + ztp_hostname: Optional[str] = None, dry_run: bool = False) -> List[dict]: + """Update linknet data for specified device using LLDP neighbor data. + """ + logger = get_logger() + result = get_neighbors(hostname=hostname)[hostname][0] + if result.failed: + raise Exception("Could not get LLDP neighbors for {}".format(hostname)) + neighbors = result.result['lldp_neighbors'] + if ztp_hostname: + settings_hostname = ztp_hostname + else: + settings_hostname = hostname + + ret = [] + + local_device_inst: Device = session.query(Device).filter(Device.hostname == hostname).one() + logger.debug("Updating linknets for device {} of type {}...".format( + local_device_inst.id, devtype.name)) + + for local_if, data in neighbors.items(): + logger.debug(f"Local: {local_if}, remote: {data[0]['hostname']} {data[0]['port']}") + remote_device_inst: Device = session.query(Device).\ + filter(Device.hostname == data[0]['hostname']).one_or_none() + if not remote_device_inst: + logger.debug(f"Unknown neighbor device, ignoring: {data[0]['hostname']}") + continue + if remote_device_inst.state in [DeviceState.DISCOVERED, DeviceState.INIT]: + # In case of MLAG init the peer does not have the correct devtype set yet, + # use same devtype as local device instead + remote_devtype = devtype + elif remote_device_inst.state not in [DeviceState.MANAGED, DeviceState.UNMANAGED]: + logger.debug("Neighbor device has invalid state, ignoring: {}".format( + data[0]['hostname'])) + continue + else: + remote_devtype = remote_device_inst.device_type + + logger.debug(f"Remote device found, device id: {remote_device_inst.id}") + + local_device_settings, _ = get_settings(settings_hostname, + devtype, + local_device_inst.model + ) + remote_device_settings, _ = get_settings(remote_device_inst.hostname, + remote_devtype, + remote_device_inst.model + ) + + verify_peer_iftype(hostname, devtype, + local_device_settings, local_if, + remote_device_inst.hostname, remote_device_inst.device_type, + remote_device_settings, data[0]['port']) + + # Check if linknet object already exists in database + local_devid = local_device_inst.id + check_linknet = session.query(Linknet).\ + filter( + ((Linknet.device_a_id == local_devid) & (Linknet.device_a_port == local_if)) + | + ((Linknet.device_b_id == local_devid) & (Linknet.device_b_port == local_if)) + | + ((Linknet.device_a_id == remote_device_inst.id) & + (Linknet.device_a_port == data[0]['port'])) + | + ((Linknet.device_b_id == remote_device_inst.id) & + (Linknet.device_b_port == data[0]['port'])) + ).one_or_none() + if check_linknet: + logger.debug(f"Found existing linknet id: {check_linknet.id}") + if ( + ( + check_linknet.device_a_id == local_devid + and check_linknet.device_a_port == local_if + and check_linknet.device_b_id == remote_device_inst.id + and check_linknet.device_b_port == data[0]['port'] + ) + or + ( + check_linknet.device_a_id == local_devid + and check_linknet.device_a_port == local_if + and check_linknet.device_b_id == remote_device_inst.id + and check_linknet.device_b_port == data[0]['port'] + ) + ): + # All info is the same, no update required + continue + else: + # TODO: update instead of delete+new insert? + if not dry_run: + session.delete(check_linknet) + session.commit() + + if devtype in [DeviceType.CORE, DeviceType.DIST] and \ + remote_device_inst.device_type in [DeviceType.CORE, DeviceType.DIST]: + ipv4_network = find_free_infra_linknet(session) + else: + ipv4_network = None + new_link = Linknet.create_linknet( + session, + hostname_a=local_device_inst.hostname, + interface_a=local_if, + hostname_b=remote_device_inst.hostname, + interface_b=data[0]['port'], + ipv4_network=ipv4_network, + strict_check=not dry_run # Don't do strict check if this is a dry_run + ) + if not dry_run: + local_device_inst.synchronized = False + remote_device_inst.synchronized = False + session.add(new_link) + session.commit() + else: + # Make sure linknet object is not added to session because of foreign key load + session.expunge(new_link) + # Make return data pretty + ret_dict = { + 'device_a_hostname': local_device_inst.hostname, + 'device_b_hostname': remote_device_inst.hostname, + **new_link.as_dict() + } + del ret_dict['id'] + del ret_dict['device_a_id'] + del ret_dict['device_b_id'] + ret.append({k: ret_dict[k] for k in sorted(ret_dict)}) + return ret diff --git a/src/cnaas_nms/db/data/default_settings.yml b/src/cnaas_nms/db/data/default_settings.yml index 09adc958..5ecad177 100644 --- a/src/cnaas_nms/db/data/default_settings.yml +++ b/src/cnaas_nms/db/data/default_settings.yml @@ -2,6 +2,6 @@ ntp_servers: [] radius_servers: [] snmp_servers: [] -evpn_spines: [] +dns_servers: [] vrfs: [] vxlans: {} diff --git a/src/cnaas_nms/db/device.py b/src/cnaas_nms/db/device.py index b3010023..81ad26a3 100644 --- a/src/cnaas_nms/db/device.py +++ b/src/cnaas_nms/db/device.py @@ -241,7 +241,11 @@ def get_mlag_peer(self, session) -> Optional[Device]: [x.hostname for x in peers] )) elif len(peers) == 1: - if self.device_type != next(iter(peers)).device_type: + peer_devtype = next(iter(peers)).device_type + if self.device_type == DeviceType.UNKNOWN or peer_devtype == DeviceType.UNKNOWN: + # Ignore check during INIT, one device might be UNKNOWN + pass + elif self.device_type != peer_devtype: raise DeviceException("MLAG peers are not the same device type") return next(iter(peers)) else: diff --git a/src/cnaas_nms/db/git.py b/src/cnaas_nms/db/git.py index 948705f2..21d3c1be 100644 --- a/src/cnaas_nms/db/git.py +++ b/src/cnaas_nms/db/git.py @@ -12,7 +12,7 @@ from cnaas_nms.db.exceptions import ConfigException, RepoStructureException from cnaas_nms.tools.log import get_logger from cnaas_nms.db.settings import get_settings, SettingsSyntaxError, DIR_STRUCTURE, \ - check_settings_collisions, VlanConflictError + check_settings_collisions, get_model_specific_configfiles, VlanConflictError from cnaas_nms.db.device import Device, DeviceType from cnaas_nms.db.session import sqla_session, redis_session from cnaas_nms.db.job import Job, JobStatus @@ -178,6 +178,10 @@ def _refresh_repo_task(repo_type: RepoType = RepoType.TEMPLATES) -> str: if not Device.valid_hostname(hostname): continue get_settings(hostname) + for devtype_str, device_models in get_model_specific_configfiles(True).items(): + devtype = DeviceType[devtype_str] + for device_model in device_models: + get_settings('nonexisting', devtype, device_model) check_settings_collisions() except SettingsSyntaxError as e: logger.exception("Error in settings repo configuration: {}".format(str(e))) diff --git a/src/cnaas_nms/db/helper.py b/src/cnaas_nms/db/helper.py index e603832a..fdbd478d 100644 --- a/src/cnaas_nms/db/helper.py +++ b/src/cnaas_nms/db/helper.py @@ -29,7 +29,7 @@ def find_mgmtdomain(session, hostnames: List[str]) -> Optional[Mgmtdomain]: """ if not isinstance(hostnames, list) or not len(hostnames) == 2: raise ValueError( - "hostnames argument must be a list with two device hostnames, got: {}".format( + "Two uplink devices are required to find a compatible mgmtdomain, got: {}".format( hostnames )) for hostname in hostnames: diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py index 64eb7d0c..a40cfb3c 100644 --- a/src/cnaas_nms/db/job.py +++ b/src/cnaas_nms/db/job.py @@ -37,6 +37,7 @@ class JobStatus(enum.Enum): FINISHED = 3 EXCEPTION = 4 ABORTED = 5 + ABORTING = 6 @classmethod def has_value(cls, value): @@ -67,6 +68,7 @@ class Job(cnaas_nms.db.base.Base): exception = Column(JSONB) finished_devices = Column(JSONB) change_score = Column(SmallInteger) # should be in range 0-100 + start_arguments = Column(JSONB) def as_dict(self) -> dict: """Return JSON serializable dict.""" @@ -84,18 +86,20 @@ def as_dict(self) -> dict: d[col.name] = value return d - def start_job(self, function_name: str, scheduled_by: str): - self.function_name = function_name + def start_job(self, function_name: Optional[str] = None, scheduled_by: Optional[str] = None): self.start_time = datetime.datetime.utcnow() self.status = JobStatus.RUNNING self.finished_devices = [] - self.scheduled_by = scheduled_by + if function_name: + self.function_name = function_name + if scheduled_by: + self.scheduled_by = scheduled_by try: json_data = json.dumps({ "job_id": self.id, "status": "RUNNING", - "function_name": function_name, - "scheduled_by": scheduled_by + "function_name": self.function_name, + "scheduled_by": self.scheduled_by }) add_event(json_data=json_data, event_type="update", update_type="job") except Exception as e: @@ -117,14 +121,17 @@ def finish_success(self, res: dict, next_job_id: Optional[int]): self.result = {"error": "unserializable"} self.finish_time = datetime.datetime.utcnow() - self.status = JobStatus.FINISHED + if self.status == JobStatus.ABORTING: + self.status = JobStatus.ABORTED + else: + self.status = JobStatus.FINISHED if next_job_id: # TODO: check if this exists in the db? self.next_job_id = next_job_id try: event_data = { "job_id": self.id, - "status": "FINISHED" + "status": self.status.name } if next_job_id: event_data['next_job_id'] = next_job_id @@ -185,6 +192,14 @@ def clear_jobs(cls, session): format(job.id)) job.status = JobStatus.ABORTED + aborting_jobs = session.query(Job).filter(Job.status == JobStatus.ABORTING).all() + job: Job + for job in aborting_jobs: + logger.warning( + "Job found in unfinished ABORTING state at startup moved to ABORTED, id: {}". + format(job.id)) + job.status = JobStatus.ABORTED + scheduled_jobs = session.query(Job).filter(Job.status == JobStatus.SCHEDULED).all() job: Job for job in scheduled_jobs: @@ -244,3 +259,13 @@ def get_previous_config(cls, session, hostname: str, previous: Optional[int] = N result['failed'] = job.result['devices'][hostname]['failed'] return result + + @classmethod + def check_job_abort_status(cls, session, job_id) -> bool: + """Check if specified job is being aborted""" + job = session.query(Job).filter(Job.id == job_id).one_or_none() + if not job: + return True + if job.status != JobStatus.RUNNING: + return True + return False diff --git a/src/cnaas_nms/db/linknet.py b/src/cnaas_nms/db/linknet.py index dd869ca8..747f1643 100644 --- a/src/cnaas_nms/db/linknet.py +++ b/src/cnaas_nms/db/linknet.py @@ -1,6 +1,7 @@ import ipaddress import enum import datetime +from typing import Optional from sqlalchemy import Column, Integer, Unicode, UniqueConstraint from sqlalchemy import ForeignKey @@ -52,40 +53,44 @@ def as_dict(self): return d @classmethod - def create_linknet(cls, session, hostname_a, interface_a, hostname_b, interface_b, linknet): - """Add a linknet between two dist/core devices.""" + def create_linknet(cls, session, hostname_a: str, interface_a: str, hostname_b: str, + interface_b: str, ipv4_network: Optional[ipaddress.IPv4Network] = None, + strict_check: bool = True): + """Add a linknet between two devices. If ipv4_network is specified both + devices must be of type CORE or DIST.""" dev_a: cnaas_nms.db.device.Device = session.query(cnaas_nms.db.device.Device).\ filter(cnaas_nms.db.device.Device.hostname == hostname_a).one_or_none() if not dev_a: raise ValueError(f"Hostname {hostname_a} not found in database") - if dev_a.state != cnaas_nms.db.device.DeviceState.MANAGED: - raise ValueError(f"Hostname {hostname_a} is not a managed device") - if dev_a.device_type not in [cnaas_nms.db.device.DeviceType.DIST, cnaas_nms.db.device.DeviceType.CORE]: - raise ValueError("Linknets can only be added between two core/dist devices (hostname_a is {})".format( - str(dev_a.device_type) - )) + if strict_check and ipv4_network and dev_a.device_type not in \ + [cnaas_nms.db.device.DeviceType.DIST, cnaas_nms.db.device.DeviceType.CORE]: + raise ValueError( + "Linknets can only be added between two core/dist devices " + + "(hostname_a is {})".format( + str(dev_a.device_type) + )) dev_b: cnaas_nms.db.device.Device = session.query(cnaas_nms.db.device.Device).\ filter(cnaas_nms.db.device.Device.hostname == hostname_b).one_or_none() if not dev_b: raise ValueError(f"Hostname {hostname_b} not found in database") - if dev_b.state != cnaas_nms.db.device.DeviceState.MANAGED: - raise ValueError(f"Hostname {hostname_b} is not a managed device") - if dev_b.device_type not in [cnaas_nms.db.device.DeviceType.DIST, cnaas_nms.db.device.DeviceType.CORE]: - raise ValueError("Linknets can only be added between two core/dist devices (hostname_b is {})".format( - str(dev_b.device_type) - )) + if strict_check and ipv4_network and dev_b.device_type not in \ + [cnaas_nms.db.device.DeviceType.DIST, cnaas_nms.db.device.DeviceType.CORE]: + raise ValueError( + "Linknets can only be added between two core/dist devices " + + "(hostname_b is {})".format( + str(dev_b.device_type) + )) - if not isinstance(linknet, ipaddress.IPv4Network) or linknet.prefixlen != 31: - import pdb - pdb.set_trace() - raise ValueError("Linknet must be an IPv4Network with prefix length of 31") - ip_a, ip_b = linknet.hosts() new_linknet: Linknet = Linknet() - new_linknet.ipv4_network = str(linknet) new_linknet.device_a = dev_a new_linknet.device_a_port = interface_a - new_linknet.device_a_ip = ip_a new_linknet.device_b = dev_b new_linknet.device_b_port = interface_b - new_linknet.device_b_ip = ip_b + if ipv4_network: + if not isinstance(ipv4_network, ipaddress.IPv4Network) or ipv4_network.prefixlen != 31: + raise ValueError("Linknet must be an IPv4Network with prefix length of 31") + ip_a, ip_b = ipv4_network.hosts() + new_linknet.device_a_ip = ip_a + new_linknet.device_b_ip = ip_b + new_linknet.ipv4_network = str(ipv4_network) return new_linknet diff --git a/src/cnaas_nms/db/mgmtdomain.py b/src/cnaas_nms/db/mgmtdomain.py index 435b80ec..f020ea53 100644 --- a/src/cnaas_nms/db/mgmtdomain.py +++ b/src/cnaas_nms/db/mgmtdomain.py @@ -15,6 +15,7 @@ from cnaas_nms.db.device import Device from cnaas_nms.db.reservedip import ReservedIP + class Mgmtdomain(cnaas_nms.db.base.Base): __tablename__ = 'mgmtdomain' __table_args__ = ( diff --git a/src/cnaas_nms/db/settings.py b/src/cnaas_nms/db/settings.py index 99e133de..83f9eced 100644 --- a/src/cnaas_nms/db/settings.py +++ b/src/cnaas_nms/db/settings.py @@ -72,6 +72,45 @@ class VlanConflictError(Exception): } } +MODEL_IF_REGEX = re.compile(r'^interfaces_(.*)\.yml$') + + +def get_model_specific_configfiles(only_modelname: bool = False) -> dict: + """Return all model specific configuration file names. + + only_modelname: only show the model name part of the filename + + Returns: + dict: dictionary with devtype as key and list of filenames as values + + { + 'CORE': [], + 'DIST': ['interfaces_veos.yml'] + } + """ + ret = {'CORE': [], 'DIST': []} + with open('/etc/cnaas-nms/repository.yml', 'r') as db_file: + repo_config = yaml.safe_load(db_file) + local_repo_path = repo_config['settings_local'] + + for devtype in ['CORE', 'DIST']: + for filename in os.listdir(os.path.join(local_repo_path, devtype.lower())): + m = re.match(MODEL_IF_REGEX, filename) + if m: + if only_modelname: + ret[devtype].append(m.groups()[0]) + else: + ret[devtype].append(filename) + return ret + + +def model_name_sanitize(model_name: str): + """Return the model name sanitized for filename purposes, + strip whitespace, convert to lowercase etc.""" + ret_name = model_name.strip().rstrip().lower() + ret_name = '_'.join(ret_name.split()) + return ret_name + def verify_dir_structure(path: str, dir_structure: dict): """Verify that given path complies to given directory structure. @@ -134,6 +173,8 @@ def get_setting_filename(repo_root: str, path: List[str]) -> str: raise ValueError("Invalid directory structure for devices settings") if not keys_exists(DIR_STRUCTURE_HOST, path[2:]): raise ValueError("File {} not defined in DIR_STRUCTURE".format(path[2:])) + elif re.match(MODEL_IF_REGEX, path[1]): + pass elif not keys_exists(DIR_STRUCTURE, path): raise ValueError("File {} not defined in DIR_STRUCTURE".format(path)) return os.path.join(repo_root, *path) @@ -248,6 +289,19 @@ def check_settings_collisions(unique_vlans: bool = True): check_vlan_collisions(devices_dict, mgmt_vlans, unique_vlans) +def get_internal_vlan_range(settings) -> range: + if "internal_vlans" not in settings or not isinstance(settings["internal_vlans"], dict): + return range(0) + if ("vlan_id_low" in settings["internal_vlans"] and + "vlan_id_high" in settings["internal_vlans"] and + type(settings["internal_vlans"]["vlan_id_low"]) == int and + type(settings["internal_vlans"]["vlan_id_high"]) == int): + return range(settings["internal_vlans"]["vlan_id_low"], + settings["internal_vlans"]["vlan_id_high"]+1) + else: + return range(0) + + def check_vlan_collisions(devices_dict: Dict[str, dict], mgmt_vlans: Set[int], unique_vlans: bool = True): logger = get_logger() @@ -297,6 +351,11 @@ def check_vlan_collisions(devices_dict: Dict[str, dict], mgmt_vlans: Set[int], device_vlan_ids[hostname].add(vxlan_data['vlan_id']) else: device_vlan_ids[hostname] = {vxlan_data['vlan_id']} + if vxlan_data['vlan_id'] in get_internal_vlan_range(settings): + raise VlanConflictError( + "VLAN id {} is overlapping with internal VLAN range".format( + vxlan_data['vlan_id']) + ) global_vlans[vxlan_data['vlan_id']] = vxlan_name # VLAN name checks if 'vlan_name' not in vxlan_data or not isinstance(vxlan_data['vlan_name'], str): @@ -442,8 +501,8 @@ def get_downstream_dependencies(hostname: str, settings: dict) -> dict: @redis_lru_cache -def get_settings(hostname: Optional[str] = None, device_type: Optional[DeviceType] = None) -> \ - Tuple[dict, dict]: +def get_settings(hostname: Optional[str] = None, device_type: Optional[DeviceType] = None, + device_model: Optional[str] = None) -> Tuple[dict, dict]: """Get settings to use for device matching hostname or global settings if no hostname is specified.""" logger = get_logger() @@ -507,6 +566,19 @@ def get_settings(hostname: Optional[str] = None, device_type: Optional[DeviceTyp local_repo_path, ['devices', hostname, 'routing.yml'], 'device->{}->routing.yml'.format(hostname), settings, settings_origin, groups) + # Check for model specific default interface settings + elif (device_type == DeviceType.DIST or device_type == DeviceType.CORE) and \ + device_type and device_model and os.path.isfile( + os.path.join(local_repo_path, + device_type.name.lower(), + 'interfaces_{}.yml'.format(model_name_sanitize(device_model)))): + settings, settings_origin = read_settings( + local_repo_path, [device_type.name.lower(), + 'interfaces_{}.yml'.format(device_model.lower())], + '{}->interfaces_{}.yml'.format(device_type.name.lower(), + model_name_sanitize(device_model)), + settings, settings_origin) + else: # Some settings parsing require knowledge of group memberships groups = [] @@ -569,3 +641,22 @@ def get_groups(hostname=''): continue groups.append(group['group']['name']) return groups + + +def get_group_regex(group_name: str) -> Optional[str]: + """Returns a string containing the regex defining the specified + group name if it's found.""" + settings, origin = get_group_settings() + if settings is None: + return None + if 'groups' not in settings: + return None + if settings['groups'] is None: + return None + for group in settings['groups']: + if 'name' not in group['group']: + continue + if 'regex' not in group['group']: + continue + if group_name == group['group']['name']: + return group['group']['regex'] diff --git a/src/cnaas_nms/db/settings_fields.py b/src/cnaas_nms/db/settings_fields.py index a15ba264..1779a23a 100644 --- a/src/cnaas_nms/db/settings_fields.py +++ b/src/cnaas_nms/db/settings_fields.py @@ -20,7 +20,7 @@ r':((:[0-9a-fA-F]{1,4}){1,7}|:))' ) FQDN_REGEX = r'([a-z0-9-]{1,63}\.)([a-z-][a-z0-9-]{1,62}\.?)+' -HOST_REGEX = f"^({IPV4_REGEX}|{FQDN_REGEX})$" +HOST_REGEX = f"^({IPV4_REGEX}|{IPV6_REGEX}|{FQDN_REGEX})$" HOSTNAME_REGEX = r"^([a-z0-9-]{1,63})(\.[a-z0-9-]{1,63})*$" host_schema = Field(..., regex=HOST_REGEX, max_length=253, description="Hostname, FQDN or IP address") @@ -34,7 +34,7 @@ ipv6_schema = Field(..., regex=f"^{IPV6_REGEX}$", description="IPv6 address") IPV6_IF_REGEX = f"{IPV6_REGEX}" + r"\/[0-9]{1,3}" -ipv6_if_schema = Field(..., regex=f"^{IPV6_IF_REGEX}$", +ipv6_if_schema = Field(None, regex=f"^{IPV6_IF_REGEX}$", description="IPv6 address in CIDR/prefix notation (::/0)") # VLAN name is alphanumeric max 32 chars on Cisco @@ -52,10 +52,12 @@ IFNAME_REGEX = r'([a-zA-Z0-9\/\.:-])+' ifname_schema = Field(None, regex=f"^{IFNAME_REGEX}$", description="Interface name") -IFCLASS_REGEX = r'(custom|downlink|uplink)' +IFCLASS_REGEX = r'(custom|downlink|fabric)' ifclass_schema = Field(None, regex=f"^{IFCLASS_REGEX}$", description="Interface class: custom, downlink or uplink") tcpudp_port_schema = Field(None, ge=0, lt=65536, description="TCP or UDP port number, 0-65535") +ebgp_multihop_schema = Field(None, ge=1, le=255, description="Numeric IP TTL, 1-255") +maximum_routes_schema = Field(None, ge=0, le=4294967294, description="Maximum number of routes to receive from peer") GROUP_NAME = r'^([a-zA-Z0-9_]{1,63}\.?)+$' group_name = Field(..., regex=GROUP_NAME, max_length=253) @@ -87,6 +89,10 @@ class f_snmp_server(BaseModel): host: str = host_schema +class f_dns_server(BaseModel): + host: str = host_schema + + class f_dhcp_relay(BaseModel): host: str = host_schema @@ -150,6 +156,14 @@ class f_extroute_bgp_neighbor_v4(BaseModel): route_map_in: str = vlan_name_schema route_map_out: str = vlan_name_schema description: str = "undefined" + bfd: Optional[bool] = None + graceful_restart: Optional[bool] = None + next_hop_self: Optional[bool] = None + update_source: Optional[str] = ifname_schema + ebgp_multihop: Optional[int] = ebgp_multihop_schema + maximum_routes: Optional[int] = maximum_routes_schema + auth_type: Optional[str] = None + auth_string: Optional[str] = None cli_append_str: str = "" @@ -159,18 +173,39 @@ class f_extroute_bgp_neighbor_v6(BaseModel): route_map_in: str = vlan_name_schema route_map_out: str = vlan_name_schema description: str = "undefined" + bfd: Optional[bool] = None + graceful_restart: Optional[bool] = None + next_hop_self: Optional[bool] = None + update_source: Optional[str] = ifname_schema + ebgp_multihop: Optional[int] = ebgp_multihop_schema + maximum_routes: Optional[int] = maximum_routes_schema + auth_type: Optional[str] = None + auth_string: Optional[str] = None cli_append_str: str = "" class f_extroute_bgp_vrf(BaseModel): name: str local_as: int = as_num_schema - neighbor_v4: Optional[List[f_extroute_bgp_neighbor_v4]] - neighbor_v6: Optional[List[f_extroute_bgp_neighbor_v6]] + neighbor_v4: List[f_extroute_bgp_neighbor_v4] = [] + neighbor_v6: List[f_extroute_bgp_neighbor_v6] = [] class f_extroute_bgp(BaseModel): - vrfs: List[f_extroute_bgp_vrf] + vrfs: List[f_extroute_bgp_vrf] = [] + + +class f_internal_vlans(BaseModel): + vlan_id_low: int = vlan_id_schema + vlan_id_high: int = vlan_id_schema + allocation_order: str = "ascending" + + @validator('vlan_id_high') + def vlan_id_high_greater_than_low(cls, v, values, **kwargs): + if v: + if values['vlan_id_low'] >= v: + raise ValueError("vlan_id_high must be greater than vlan_id_low") + return v class f_vxlan(BaseModel): @@ -180,8 +215,10 @@ class f_vxlan(BaseModel): vlan_id: int = vlan_id_schema vlan_name: str = vlan_name_schema ipv4_gw: Optional[str] = ipv4_if_schema + ipv6_gw: Optional[str] = ipv6_if_schema dhcp_relays: Optional[List[f_dhcp_relay]] mtu: Optional[int] = mtu_schema + vxlan_host_route: bool = True groups: List[str] = [] devices: List[str] = [] @@ -192,6 +229,13 @@ def vrf_required_if_ipv4_gw_set(cls, v, values, **kwargs): raise ValueError('VRF is required when specifying ipv4_gw') return v + @validator('ipv6_gw') + def vrf_required_if_ipv6_gw_set(cls, v, values, **kwargs): + if v: + if 'vrf' not in values or not values['vrf']: + raise ValueError('VRF is required when specifying ipv6_gw') + return v + class f_underlay(BaseModel): infra_lo_net: str = ipv4_if_schema @@ -204,6 +248,7 @@ class f_root(BaseModel): radius_servers: List[f_radius_server] = [] syslog_servers: List[f_syslog_server] = [] snmp_servers: List[f_snmp_server] = [] + dns_servers: List[f_dns_server] = [] dhcp_relays: Optional[List[f_dhcp_relay]] interfaces: List[f_interface] = [] vrfs: List[f_vrf] = [] @@ -213,6 +258,7 @@ class f_root(BaseModel): extroute_static: Optional[f_extroute_static] extroute_ospfv3: Optional[f_extroute_ospfv3] extroute_bgp: Optional[f_extroute_bgp] + internal_vlans: Optional[f_internal_vlans] cli_prepend_str: str = "" cli_append_str: str = "" diff --git a/src/cnaas_nms/run.py b/src/cnaas_nms/run.py index a73f0b0b..77ae2740 100644 --- a/src/cnaas_nms/run.py +++ b/src/cnaas_nms/run.py @@ -3,7 +3,6 @@ import atexit import signal import threading -import time from typing import List from gevent import monkey, signal as gevent_signal from redis import StrictRedis @@ -13,6 +12,7 @@ os.environ['PYTHONPATH'] = os.getcwd() +stop_websocket_threads = False print("Code coverage collection for worker in pid {}: {}".format( @@ -112,6 +112,8 @@ def thread_websocket_events(): if not event: continue emit_redis_event(event) + if stop_websocket_threads: + break if __name__ == '__main__': @@ -127,9 +129,13 @@ def thread_websocket_events(): apidata = get_apidata() if isinstance(apidata, dict) and 'host' in apidata: - app.socketio.run(get_app(), debug=True, host=apidata['host']) + host = apidata['host'] else: - app.socketio.run(get_app(), debug=True) + host = None + + app.socketio.run(get_app(), debug=True, host=host) + stop_websocket_threads = True + t_websocket_events.join() if 'COVERAGE' in os.environ: save_coverage() diff --git a/src/cnaas_nms/scheduler/scheduler.py b/src/cnaas_nms/scheduler/scheduler.py index 697164b4..65989ad6 100644 --- a/src/cnaas_nms/scheduler/scheduler.py +++ b/src/cnaas_nms/scheduler/scheduler.py @@ -7,7 +7,7 @@ from typing import Optional, Union from types import FunctionType -from apscheduler.schedulers.background import BackgroundScheduler, BlockingScheduler +from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.executors.pool import ThreadPoolExecutor @@ -112,17 +112,50 @@ def shutdown(self): if self._scheduler and not self.use_mule: return self._scheduler.shutdown() - def add_job(self, func, **kwargs): + def add_local_job(self, func, **kwargs): + """Add job to local scheduler.""" return self._scheduler.add_job(func, **kwargs) + def remove_local_job(self, job_id): + """Remove job from local scheduler.""" + return self._scheduler.remove_job(str(job_id)) + + def remove_scheduled_job(self, job_id, abort_message="removed"): + """Remove scheduled job from mule worker or local scheduler depending + on setup.""" + if self.use_mule: + try: + import uwsgi + except Exception as e: + logger.exception("use_mule is set but not running in uwsgi") + raise e + args = { + "scheduler_action": "remove", + "id": str(job_id) + } + uwsgi.mule_msg(json.dumps(args)) + else: + self.remove_local_job(job_id) + + with sqla_session() as session: + job = session.query(Job).filter(Job.id == job_id).one_or_none() + job.finish_abort(message=abort_message) + def add_onetime_job(self, func: Union[str, FunctionType], when: Optional[int] = None, scheduled_by: Optional[str] = None, **kwargs) -> int: - """Schedule a job to run at a later time. + """Schedule a job to run at a later time on the mule worker or + local scheduler depending on setup. + + Some extra checks against kwargs are performed here. If kwarg + with name 'dry_run' is included, (dry_run) is appended to function + name. If kwarg job_comment or job_ticket_ref are included, those + fields in the job will be populated. Args: func: The function to call when: Optional number of seconds to wait before starting job + scheduled_by: Username that scheduled the job **kwargs: Arguments to pass through to called function Returns: int: job_id @@ -134,10 +167,39 @@ def add_onetime_job(self, func: Union[str, FunctionType], trigger = None run_date = None + if isinstance(func, FunctionType): + func_qualname = str(func.__qualname__) + else: + func_qualname = str(func) + func_name = func_qualname.split(':')[-1] + + try: + json.dumps(kwargs) + except TypeError as e: + raise TypeError("Job args must be JSON serializable: {}".format(e)) + + # Append (dry_run) to function name if set, so we can distinguish dry_run jobs + try: + if kwargs['kwargs']['dry_run']: + func_name += " (dry_run)" + except Exception: + pass + with sqla_session() as session: job = Job() if run_date: job.scheduled_time = run_date + job.function_name = func_name + if scheduled_by is None: + scheduled_by = 'unknown' + job.scheduled_by = scheduled_by + job_comment = kwargs['kwargs'].pop('job_comment', None) + if job_comment and isinstance(job_comment, str): + job.comment = job_comment[:255] + job_ticket_ref = kwargs['kwargs'].pop('job_ticket_ref', None) + if job_ticket_ref and isinstance(job_comment, str): + job.ticket_ref = job_ticket_ref[:32] + job.start_arguments = kwargs['kwargs'] session.add(job) session.flush() job_id = job.id @@ -151,17 +213,14 @@ def add_onetime_job(self, func: Union[str, FunctionType], logger.exception("use_mule is set but not running in uwsgi") raise e args = dict(kwargs) - if isinstance(func, FunctionType): - args['func'] = str(func.__qualname__) - else: - args['func'] = str(func) + args['func'] = func_qualname args['trigger'] = trigger args['when'] = when args['id'] = str(job_id) uwsgi.mule_msg(json.dumps(args)) return job_id else: - self._scheduler.add_job(func, trigger=trigger, kwargs=kwargs, - id=str(job_id), - run_date=run_date) + self.add_local_job(func, trigger=trigger, kwargs=kwargs, id=str(job_id), + run_date=run_date, name=func_qualname) return job_id + diff --git a/src/cnaas_nms/scheduler/tests/test_scheduler.py b/src/cnaas_nms/scheduler/tests/test_scheduler.py index 7eedb094..0a45b08b 100644 --- a/src/cnaas_nms/scheduler/tests/test_scheduler.py +++ b/src/cnaas_nms/scheduler/tests/test_scheduler.py @@ -7,10 +7,12 @@ from cnaas_nms.scheduler.scheduler import Scheduler from cnaas_nms.scheduler.wrapper import job_wrapper from cnaas_nms.scheduler.jobresult import DictJobResult +from cnaas_nms.db.session import sqla_session +from cnaas_nms.db.job import Job, JobStatus @job_wrapper -def testfunc_success(text=''): +def testfunc_success(text='', job_id=None, scheduled_by=None): print(text) return DictJobResult( result = {'status': 'success'} @@ -18,26 +20,29 @@ def testfunc_success(text=''): @job_wrapper -def testfunc_exception(text=''): +def testfunc_exception(text='', job_id=None, scheduled_by=None): print(text) raise Exception("testfunc_exception raised exception") class InitTests(unittest.TestCase): - def setUp(self): - data_dir = pkg_resources.resource_filename(__name__, 'data') - with open(os.path.join(data_dir, 'testdata.yml'), 'r') as f_testdata: - self.testdata = yaml.safe_load(f_testdata) - + @classmethod + def setUpClass(cls) -> None: scheduler = Scheduler() scheduler.start() - def tearDown(self): + @classmethod + def tearDownClass(cls) -> None: scheduler = Scheduler() time.sleep(3) scheduler.get_scheduler().print_jobs() scheduler.shutdown() + def setUp(self): + data_dir = pkg_resources.resource_filename(__name__, 'data') + with open(os.path.join(data_dir, 'testdata.yml'), 'r') as f_testdata: + self.testdata = yaml.safe_load(f_testdata) + def test_add_schedule(self): scheduler = Scheduler() job1_id = scheduler.add_onetime_job(testfunc_success, when=1, @@ -46,10 +51,35 @@ def test_add_schedule(self): job2_id = scheduler.add_onetime_job(testfunc_exception, when=1, scheduled_by='test_user', kwargs={'text': 'exception'}) - assert isinstance(job1_id, str) - assert isinstance(job2_id, str) - print(f"Job1 scheduled as ID { job1_id }") - print(f"Job2 scheduled as ID { job2_id }") + assert isinstance(job1_id, int) + assert isinstance(job2_id, int) + print(f"Test job 1 scheduled as ID { job1_id }") + print(f"Test job 2 scheduled as ID { job2_id }") + time.sleep(3) + with sqla_session() as session: + job1 = session.query(Job).filter(Job.id == job1_id).one_or_none() + self.assertIsInstance(job1, Job, "Test job 1 could not be found") + self.assertEqual(job1.status, JobStatus.FINISHED, "Test job 1 did not finish") + self.assertEqual(job1.result, {'status': 'success'}, "Test job 1 returned bad status") + job2 = session.query(Job).filter(Job.id == job2_id).one_or_none() + self.assertIsInstance(job2, Job, "Test job 2 could not be found") + self.assertEqual(job2.status, JobStatus.EXCEPTION, "Test job 2 did not make exception") + self.assertIn("message", job2.exception, "Test job 2 did not contain message in exception") + + def test_abort_schedule(self): + scheduler = Scheduler() + job3_id = scheduler.add_onetime_job(testfunc_success, when=600, + scheduled_by='test_user', + kwargs={'text': 'abort'}) + assert isinstance(job3_id, int) + print(f"Test job 3 scheduled as ID { job3_id }") + scheduler.remove_scheduled_job(job3_id) + time.sleep(3) + with sqla_session() as session: + job3 = session.query(Job).filter(Job.id == job3_id).one_or_none() + self.assertIsInstance(job3, Job, "Test job 3 could not be found") + self.assertEqual(job3.status, JobStatus.ABORTED, "Test job 3 did not abort") + self.assertEqual(job3.result, {'message': 'removed'}, "Test job 3 returned bad status") if __name__ == '__main__': diff --git a/src/cnaas_nms/scheduler/wrapper.py b/src/cnaas_nms/scheduler/wrapper.py index 6bca54d1..94efb2c2 100644 --- a/src/cnaas_nms/scheduler/wrapper.py +++ b/src/cnaas_nms/scheduler/wrapper.py @@ -50,7 +50,7 @@ def update_device_progress_thread(stop_event: threading.Event, job_id: int): def job_wrapper(func): """Decorator to save job status in job tracker database.""" - def wrapper(job_id: int, scheduled_by: str, *args, **kwargs): + def wrapper(job_id: int, scheduled_by: str, kwargs={}): if not job_id or not type(job_id) == int: errmsg = "Missing job_id when starting job for {}".format(func.__name__) logger.error(errmsg) @@ -62,25 +62,12 @@ def wrapper(job_id: int, scheduled_by: str, *args, **kwargs): errmsg = "Could not find job_id {} in database".format(job_id) logger.error(errmsg) raise ValueError(errmsg) - kwargs['kwargs']['job_id'] = job_id - if scheduled_by is None: - scheduled_by = 'unknown' - # Append (dry_run) to function name if set, so we can distinguish dry_run jobs - try: - if kwargs['kwargs']['dry_run']: - function_name = "{} (dry_run)".format(func.__name__) - else: - function_name = func.__name__ - except Exception: + kwargs['job_id'] = job_id + # Don't send new function name unless it was set to "wrapper" + function_name = None + if job.function_name == "wrapper": function_name = func.__name__ - job_comment = kwargs['kwargs'].pop('job_comment', None) - if job_comment and isinstance(job_comment, str): - job.comment = job_comment[:255] - job_ticket_ref = kwargs['kwargs'].pop('job_ticket_ref', None) - if job_ticket_ref and isinstance(job_comment, str): - job.ticket_ref = job_ticket_ref[:32] - job.start_job(function_name=function_name, - scheduled_by=scheduled_by) + job.start_job(function_name=function_name) if func.__name__ in progress_funcitons: stop_event = threading.Event() device_thread = threading.Thread(target=update_device_progress_thread, @@ -89,7 +76,7 @@ def wrapper(job_id: int, scheduled_by: str, *args, **kwargs): try: set_thread_data(job_id) # kwargs is contained in an item called kwargs because of the apscheduler.add_job call - res = func(*args, **kwargs['kwargs']) + res = func(**kwargs) if job_id: res = insert_job_id(res, job_id) del thread_data.job_id diff --git a/src/cnaas_nms/scheduler_mule.py b/src/cnaas_nms/scheduler_mule.py index 627b2b43..e7bfa144 100644 --- a/src/cnaas_nms/scheduler_mule.py +++ b/src/cnaas_nms/scheduler_mule.py @@ -38,9 +38,9 @@ def pre_schedule_checks(scheduler, kwargs): for job in scheduler.get_scheduler().get_jobs(): # Only allow scheduling of one discover_device job at the same time if job.name == 'cnaas_nms.confpush.init_device:discover_device': - if job.kwargs['kwargs']['ztp_mac'] == kwargs['kwargs']['ztp_mac']: - message = ("There is already another scheduled job to discover device {}, skipping ". - format(kwargs['kwargs']['ztp_mac'])) + if job.kwargs['kwargs']['dhcp_ip'] == kwargs['kwargs']['dhcp_ip']: + message = ("There is already another scheduled job to discover {} {}, skipping ". + format(kwargs['kwargs']['ztp_mac'], kwargs['kwargs']['dhcp_ip'])) check_ok = False if not check_ok: @@ -76,22 +76,29 @@ def main_loop(): while True: mule_data = uwsgi.mule_get_msg() data: dict = json.loads(mule_data) - if data['when'] and isinstance(data['when'], int): + action = "add" + if 'scheduler_action' in data: + if data['scheduler_action'] == "remove": + action = "remove" + if 'when' in data and isinstance(data['when'], int): data['run_date'] = datetime.datetime.utcnow() + datetime.timedelta(seconds=data['when']) del data['when'] kwargs = {} for k, v in data.items(): - if k not in ['func', 'trigger', 'id', 'run_date']: + if k not in ['func', 'trigger', 'id', 'run_date', 'scheduler_action']: kwargs[k] = v # Perform pre-schedule job checks try: - if not pre_schedule_checks(scheduler, kwargs): + if action == "add" and not pre_schedule_checks(scheduler, kwargs): continue except Exception as e: logger.exception("Unable to perform pre-schedule job checks: {}".format(e)) - scheduler.add_job(data['func'], trigger=data['trigger'], kwargs=kwargs, - id=data['id'], run_date=data['run_date'], name=data['func']) + if action == "add": + scheduler.add_local_job(data['func'], trigger=data['trigger'], kwargs=kwargs, + id=data['id'], run_date=data['run_date'], name=data['func']) + elif action == "remove": + scheduler.remove_local_job(data['id']) if __name__ == '__main__': diff --git a/src/cnaas_nms/tools/pki.py b/src/cnaas_nms/tools/pki.py new file mode 100644 index 00000000..c5d54902 --- /dev/null +++ b/src/cnaas_nms/tools/pki.py @@ -0,0 +1,106 @@ +import ssl +import os +import datetime +from ipaddress import IPv4Address + +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from cnaas_nms.tools.get_apidata import get_apidata +from cnaas_nms.tools.log import get_logger + +logger = get_logger() + + +def get_ssl_context(): + apidata = get_apidata() + new_ssl_context = None + + if 'cafile' in apidata: + if os.path.isfile(apidata['cafile']): + new_ssl_context = ssl.create_default_context(cafile=apidata['cafile']) + else: + logger.error("Specified cafile is not a file: {}".format(apidata['cafile'])) + + if 'verify_tls_device' in apidata and type(apidata['verify_tls_device']) == bool and \ + not apidata['verify_tls_device']: + logger.warning("Accepting unverified TLS certificates") + new_ssl_context = ssl._create_unverified_context() + + if not new_ssl_context: + logger.debug("Using system default CAs") + new_ssl_context = ssl.create_default_context() + + return new_ssl_context + + +def generate_device_cert(hostname: str, ipv4_address: IPv4Address): + apidata = get_apidata() + try: + if not os.path.isfile(apidata['cafile']): + raise Exception("Specified cafile is not a file: {}".format(apidata['cafile'])) + except KeyError: + raise Exception("No cafile specified in api.yml") + + try: + if not os.path.isfile(apidata['cakeyfile']): + raise Exception("Specified cakeyfile is not a file: {}".format(apidata['cakeyfile'])) + except KeyError: + raise Exception("No cakeyfile specified in api.yml") + + try: + if not os.path.isdir(apidata['certpath']): + raise Exception("Specified certpath is not a directory") + except KeyError: + raise Exception("No certpath found in api.yml settings") + + with open(apidata['cakeyfile'], "rb") as cakeyfile: + root_key = serialization.load_pem_private_key( + cakeyfile.read(), + password=None, + ) + + with open(apidata['cafile'], "rb") as cafile: + root_cert = x509.load_pem_x509_certificate( + cafile.read() + ) + + cert_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + new_subject = x509.Name( + [ + x509.NameAttribute(NameOID.COMMON_NAME, hostname), + ] + ) + cert = ( + x509.CertificateBuilder() + .subject_name(new_subject) + .issuer_name(root_cert.issuer) + .public_key(cert_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=7300)) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipv4_address)]), + critical=False, + ) + .sign(root_key, hashes.SHA256(), default_backend()) + ) + + with open(os.path.join(apidata['certpath'], "{}.crt".format(hostname)), "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + + with open(os.path.join(apidata['certpath'], "{}.key".format(hostname)), "wb") as f: + f.write(cert_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + )) + + +ssl_context = get_ssl_context() diff --git a/src/cnaas_nms/tools/template_dry_run.py b/src/cnaas_nms/tools/template_dry_run.py index f4dc1364..f0d0ff01 100755 --- a/src/cnaas_nms/tools/template_dry_run.py +++ b/src/cnaas_nms/tools/template_dry_run.py @@ -48,9 +48,13 @@ def get_device_details(hostname): def render_template(platform, device_type, variables): + # Jinja env should match nornir_helper.cnaas_ninja_env jinjaenv = jinja2.Environment( loader=jinja2.FileSystemLoader(platform), - undefined=jinja2.StrictUndefined, trim_blocks=True + undefined=jinja2.StrictUndefined, + trim_blocks=True, + lstrip_blocks=True, + keep_trailing_newline=True ) template_secrets = {} for env in os.environ: diff --git a/src/cnaas_nms/version.py b/src/cnaas_nms/version.py index a676d27c..65397202 100644 --- a/src/cnaas_nms/version.py +++ b/src/cnaas_nms/version.py @@ -1,3 +1,3 @@ -__version__ = '1.1.0' +__version__ = '1.2.0' __version_info__ = tuple([field for field in __version__.split('.')]) __api_version__ = 'v1.0' diff --git a/test/integrationtests.py b/test/integrationtests.py index 0b8a19f0..7e263fa4 100644 --- a/test/integrationtests.py +++ b/test/integrationtests.py @@ -4,6 +4,7 @@ import time import unittest import os +import datetime if 'CNAASURL' in os.environ: @@ -131,10 +132,23 @@ def test_02_ztp(self): hostname, device_id = self.wait_for_discovered_device() print("Discovered hostname, id: {}, {}".format(hostname, device_id)) self.assertTrue(hostname, "No device in state discovered found for ZTP") + data = {"hostname": "eosaccess", "device_type": "ACCESS"} + r = requests.post( + f'{URL}/api/v1.0/device_initcheck/{device_id}', + headers=AUTH_HEADER, + json=data, + verify=TLS_VERIFY + ) + self.assertEqual(r.status_code, 200, "Failed device_initcheck, http status") + self.assertEqual(r.json()['status'], 'success', + "Failed device_initcheck, returned unsuccessful") +# this check fails when running integrationtests with one dist because of faked neighbor: +# self.assertTrue(r.json()['data']['compatible'], "initcheck was not compatible") + r = requests.post( f'{URL}/api/v1.0/device_init/{device_id}', headers=AUTH_HEADER, - json={"hostname": "eosaccess", "device_type": "ACCESS"}, + json=data, verify=TLS_VERIFY ) self.assertEqual(r.status_code, 200, "Failed to start device_init") @@ -190,7 +204,10 @@ def test_04_syncto_access(self): verify=TLS_VERIFY ) self.assertEqual(r.status_code, 200, "Failed to do sync_to access") - self.check_jobid(r.json()['job_id']) + auto_job1 = self.check_jobid(r.json()['job_id']) + self.assertEqual(type(auto_job1['next_job_id']), int, "No auto-push commit job found") + self.check_jobid(auto_job1['next_job_id']) + def test_05_syncto_dist(self): r = requests.post( @@ -233,7 +250,7 @@ def test_08_firmware(self): verify=TLS_VERIFY ) # TODO: not working - #self.assertEqual(r.status_code, 200, "Failed to list firmware") + self.assertEqual(r.status_code, 200, "Failed to list firmware") def test_09_sysversion(self): r = requests.get( @@ -277,9 +294,128 @@ def test_11_update_facts_dist(self): verify=TLS_VERIFY ) self.assertEqual(r.status_code, 200, "Failed to do update facts for dist") + update_facts_job_id = r.json()['job_id'] + job = self.check_jobid(update_facts_job_id) + self.assertIn("diff", job['result']) + + def test_12_abort_running_job(self): + data = { + 'group': 'DIST', + 'url': '', + 'post_flight': True, + 'post_waittime': 30 + } + result = requests.post( + f"{URL}/api/v1.0/firmware/upgrade", + headers=AUTH_HEADER, + json=data, + verify=TLS_VERIFY + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json()['status'], 'success') + self.assertEqual(type(result.json()['job_id']), int) + job_id: int = result.json()['job_id'] + time.sleep(2) + result = requests.get( + f'{URL}/api/v1.0/job/{job_id}', + headers=AUTH_HEADER, + verify=TLS_VERIFY + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json()['status'], 'success') + self.assertEqual(len(result.json()['data']['jobs']), 1, "One job should be found") + self.assertEqual(result.json()['data']['jobs'][0]['status'], "RUNNING", + "Job should be in RUNNING state at start") + abort_data = { + 'action': 'ABORT', + 'abort_reason': 'unit test abort_running_job' + } + result = requests.put( + f"{URL}/api/v1.0/job/{job_id}", + headers=AUTH_HEADER, + json=abort_data, + verify=TLS_VERIFY + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json()['status'], 'success') + result = requests.get( + f'{URL}/api/v1.0/job/{job_id}', + headers=AUTH_HEADER, + verify=TLS_VERIFY + ) + self.assertEqual(result.json()['data']['jobs'][0]['status'], "ABORTING", + "Job should be in ABORTING state after abort action") + time.sleep(30) + result = requests.get( + f'{URL}/api/v1.0/job/{job_id}', + headers=AUTH_HEADER, + verify=TLS_VERIFY + ) + self.assertEqual(result.json()['data']['jobs'][0]['status'], "ABORTED", + "Job should be in ABORTED state at end") + + def test_13_abort_scheduled_job(self): + start_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=30) + data = { + 'group': 'DIST', + 'url': '', + 'post_flight': True, + 'post_waittime': 30, + 'start_at': start_time.strftime('%Y-%m-%d %H:%M:%S') + } + result = requests.post( + f"{URL}/api/v1.0/firmware/upgrade", + headers=AUTH_HEADER, + json=data, + verify=TLS_VERIFY + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json()['status'], 'success') + self.assertEqual(type(result.json()['job_id']), int) + job_id: int = result.json()['job_id'] + time.sleep(2) + result = requests.get( + f'{URL}/api/v1.0/job/{job_id}', + headers=AUTH_HEADER, + verify=TLS_VERIFY + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json()['status'], 'success') + self.assertEqual(len(result.json()['data']['jobs']), 1, "One job should be found") + self.assertEqual(result.json()['data']['jobs'][0]['status'], "SCHEDULED", + "Job should be in SCHEDULED state at start") + abort_data = { + 'action': 'ABORT', + 'abort_reason': 'unit test abort_scheduled_job' + } + result = requests.put( + f"{URL}/api/v1.0/job/{job_id}", + headers=AUTH_HEADER, + json=abort_data, + verify=TLS_VERIFY + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json()['status'], 'success') + result = requests.get( + f'{URL}/api/v1.0/job/{job_id}', + headers=AUTH_HEADER, + verify=TLS_VERIFY + ) + self.assertEqual(result.json()['data']['jobs'][0]['status'], "ABORTED", + "Job should be in ABORTED state at end") + + def test_11_update_interfaces_access(self): + hostname = "eosaccess" + r = requests.post( + f'{URL}/api/v1.0/device_update_interfaces', + headers=AUTH_HEADER, + json={"hostname": hostname}, + verify=TLS_VERIFY + ) + self.assertEqual(r.status_code, 200, "Failed to do update interfaces for dist") restore_job_id = r.json()['job_id'] job = self.check_jobid(restore_job_id) - self.assertFalse(job['result']['devices'][hostname]['failed']) + self.assertEqual(job['status'], "FINISHED") if __name__ == '__main__': diff --git a/test/integrationtests.sh b/test/integrationtests.sh index 11c08294..d3868ed8 100755 --- a/test/integrationtests.sh +++ b/test/integrationtests.sh @@ -1,9 +1,9 @@ -#!/bin/bash +#!/bin/bash -e pushd . cd ../docker/ # if running selinux on host this is required: chcon -Rt svirt_sandbox_file_t coverage/ -mkdir coverage/ +mkdir -p coverage/ export GITREPO_TEMPLATES="git://gitops.sunet.se/cnaas-lab-templates" export GITREPO_SETTINGS="git://gitops.sunet.se/cnaas-lab-settings" @@ -19,8 +19,43 @@ export PASSWORD_MANAGED="abc123abc123" export COVERAGE=1 export JWT_AUTH_TOKEN="eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJpYXQiOjE1NzEwNTk2MTgsIm5iZiI6MTU3MTA1OTYxOCwianRpIjoiNTQ2MDk2YTUtZTNmOS00NzFlLWE2NTctZWFlYTZkNzA4NmVhIiwic3ViIjoiYWRtaW4iLCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.Sfffg9oZg_Kmoq7Oe8IoTcbuagpP6nuUXOQzqJpgDfqDq_GM_4zGzt7XxByD4G0q8g4gZGHQnV14TpDer2hJXw" +docker-compose down + +if docker volume ls | egrep -q "cnaas-postgres-data$" +then + if [ -z "$AUTOTEST" ] + then + read -p "Do you want to continue and reset existing SQL database? [y/N]" ans + case $ans in + [Yy]* ) docker volume rm cnaas-postgres-data;; + * ) exit 1;; + esac + else + docker volume rm cnaas-postgres-data + fi +fi + +docker volume create cnaas-templates +docker volume create cnaas-settings +docker volume create cnaas-postgres-data +docker volume create cnaas-jwtcert +docker volume create cnaas-cacert + docker-compose up -d +docker cp ./jwt-cert/public.pem docker_cnaas_api_1:/opt/cnaas/jwtcert/public.pem +docker-compose exec -T cnaas_api /bin/chown -R www-data:www-data /opt/cnaas/jwtcert/ +docker-compose exec -T cnaas_api /opt/cnaas/createca.sh + +if [ ! -z "$PRE_TEST_SCRIPT" ] +then + if [ -x "$PRE_TEST_SCRIPT" ] + then + echo "Running PRE_TEST_SCRIPT..." + bash -c $PRE_TEST_SCRIPT + fi +fi + # go back to test dir popd