From 28bfd77c23a2d8e1e83651a8feed2cd0295bfa73 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Thu, 6 Jul 2023 21:24:53 +0200 Subject: [PATCH 1/7] feat(Multi server): Add multi server support https://github.com/netdevopsbr/netbox-proxbox/issues/33 Signed-off-by: devopstales <42894256+devopstales@users.noreply.github.com> --- netbox_proxbox/__init__.py | 24 +-- netbox_proxbox/proxbox_api/create/dcim.py | 4 +- .../proxbox_api/create/virtualization.py | 17 +-- netbox_proxbox/proxbox_api/plugins_config.py | 142 +++++++++-------- netbox_proxbox/proxbox_api/remove.py | 13 +- netbox_proxbox/proxbox_api/update.py | 143 ++++++++++++++---- netbox_proxbox/proxbox_api/updates/node.py | 12 +- .../proxbox_api/updates/virtual_machine.py | 18 +-- 8 files changed, 224 insertions(+), 149 deletions(-) diff --git a/netbox_proxbox/__init__.py b/netbox_proxbox/__init__.py index 054f72f..5b409ba 100755 --- a/netbox_proxbox/__init__.py +++ b/netbox_proxbox/__init__.py @@ -11,17 +11,19 @@ class ProxboxConfig(PluginConfig): base_url = "proxbox" required_settings = [] default_settings = { - 'proxmox': { - 'domain': 'proxbox.example.com', # May also be IP address - 'http_port': 8006, - 'user': 'root@pam', - 'password': 'Strong@P4ssword', - 'token': { - 'name': 'tokenID', - 'value': '039az154-23b2-4be0-8d20-b66abc8c4686' - }, - 'ssl': False - }, + 'proxmox': [ + { + 'domain': 'proxbox.example.com', # May also be IP address + 'http_port': 8006, + 'user': 'root@pam', + 'password': 'Strong@P4ssword', + 'token': { + 'name': 'tokenID', + 'value': '039az154-23b2-4be0-8d20-b66abc8c4686' + }, + 'ssl': False + } + ], 'netbox': { 'domain': 'netbox.example.com', # May also be IP address 'http_port': 80, diff --git a/netbox_proxbox/proxbox_api/create/dcim.py b/netbox_proxbox/proxbox_api/create/dcim.py index 28cfc4a..a1f47c1 100755 --- a/netbox_proxbox/proxbox_api/create/dcim.py +++ b/netbox_proxbox/proxbox_api/create/dcim.py @@ -130,7 +130,7 @@ def site(**kwargs): # # dcim.devices (nodes) # -def node(proxmox_node): +def node(proxmox, proxmox_node): # Create json with basic NODE information node_json = {} node_json["name"] = proxmox_node['name'] @@ -140,7 +140,7 @@ def node(proxmox_node): node_json["status"] = 'active' node_json["tags"] = [extras.tag().id] - cluster = virtualization.cluster() + cluster = virtualization.cluster(proxmox) if cluster: if cluster != None: node_json["cluster"] = cluster.id diff --git a/netbox_proxbox/proxbox_api/create/virtualization.py b/netbox_proxbox/proxbox_api/create/virtualization.py index 56feaef..a092697 100755 --- a/netbox_proxbox/proxbox_api/create/virtualization.py +++ b/netbox_proxbox/proxbox_api/create/virtualization.py @@ -3,9 +3,7 @@ # PLUGIN_CONFIG variables from ..plugins_config import ( NETBOX_SESSION as nb, - PROXMOX_SESSION as proxmox, NETBOX_VM_ROLE_ID, - ) from . import ( @@ -54,7 +52,7 @@ def cluster_type(): # # virtualization.clusters # -def cluster(): +def cluster(proxmox): # # Cluster # @@ -135,19 +133,10 @@ def cluster(): except: return f"Error creating the '{proxmox_cluster_name}' cluster. Possible errors: the name '{proxmox_cluster_name}' is already used." - - - - - - - - - # # virtualization.virtual_machines # -def virtual_machine(proxmox_vm, duplicate): +def virtual_machine(proxmox, proxmox_vm, duplicate): # Create json with basic VM/CT information vm_json = {} netbox_obj = None @@ -164,7 +153,7 @@ def virtual_machine(proxmox_vm, duplicate): vm_json["name"] = proxmox_vm['name'] vm_json["status"] = 'active' - vm_json["cluster"] = cluster().id + vm_json["cluster"] = cluster(proxmox).id vm_json["role"] = extras.role(role_id = NETBOX_VM_ROLE_ID).id vm_json["tags"] = [extras.tag().id] diff --git a/netbox_proxbox/proxbox_api/plugins_config.py b/netbox_proxbox/proxbox_api/plugins_config.py index 7b2f6f6..cf1eef7 100755 --- a/netbox_proxbox/proxbox_api/plugins_config.py +++ b/netbox_proxbox/proxbox_api/plugins_config.py @@ -74,23 +74,6 @@ PROXMOX_SETTING = USER_PLUGINS_CONFIG.get("proxmox", DEFAULT_PROXMOX_SETTING) NETBOX_SETTING = USER_PLUGINS_CONFIG.get("netbox", DEFAULT_NETBOX_SETTING) -# -# Proxmox related settings -# -# API URI -PROXMOX = PROXMOX_SETTING.get("domain", DEFAULT_PROXMOX) -PROXMOX_PORT = PROXMOX_SETTING.get("http_port", DEFAULT_PROXMOX_PORT) -PROXMOX_SSL = PROXMOX_SETTING.get("ssl", DEFAULT_PROXMOX_SSL) - -# ACCESS -PROXMOX_USER = PROXMOX_SETTING.get("user", DEFAULT_PROXMOX_USER) -PROXMOX_PASSWORD = PROXMOX_SETTING.get("password", DEFAULT_PROXMOX_PASSWORD) - -PROXMOX_TOKEN = PROXMOX_SETTING.get("token", DEFAULT_PROXMOX_TOKEN) -if PROXMOX_TOKEN != None: - PROXMOX_TOKEN_NAME = PROXMOX_TOKEN.get("name", DEFAULT_PROXMOX_TOKEN_NAME) - PROXMOX_TOKEN_VALUE = PROXMOX_TOKEN.get("value", DEFAULT_PROXMOX_TOKEN_VALUE) - # # NETBOX RELATED SETTINGS # @@ -111,49 +94,88 @@ NETBOX_NODE_ROLE_ID = NETBOX_SETTINGS.get("node_role_id", DEFAULT_NETBOX_NODE_ROLE_ID) NETBOX_SITE_ID = NETBOX_SETTINGS.get("site_id", DEFAULT_NETBOX_SITE_ID) - -#################################################################################################### -# # -# WITH PLUGIN CONFIGURED, STARTS BOTH PROXMOX AND NETBOX SESSION # -# # -#################################################################################################### - -# -# PROXMOX SESSION -# -# Check if token was provided -if PROXMOX_TOKEN_VALUE != None and len(PROXMOX_TOKEN_VALUE) > 0: - try: - if PROXMOX_SSL == False: - # DISABLE SSL WARNING - import urllib3 - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - # Start PROXMOX session using TOKEN - PROXMOX_SESSION = ProxmoxAPI( - PROXMOX, - port=PROXMOX_PORT, - user=PROXMOX_USER, - token_name=PROXMOX_TOKEN_NAME, - token_value=PROXMOX_TOKEN_VALUE, - verify_ssl=PROXMOX_SSL - ) - except: - raise RuntimeError(f'Error trying to initialize Proxmox Session using TOKEN (token_name: {PROXMOX_TOKEN_NAME} and token_value: {PROXMOX_TOKEN_VALUE} provided') - -# If token not provided, start session using user and passwd -else: - try: - # Start PROXMOX session using USER CREDENTIALS - PROXMOX_SESSION = ProxmoxAPI( - PROXMOX, - port=PROXMOX_PORT, - user=PROXMOX_USER, - password=PROXMOX_PASSWORD, - verify_ssl=PROXMOX_SSL - ) - except: - raise RuntimeError(f'Error trying to initialize Proxmox Session using USER {PROXMOX_USER} and PASSWORD') +PROXMOX_SESSIONS = {} + +def get_proxmox_session(PROXMOX_SETTING): + # + # Proxmox related settings + # + # API URI + PROXMOX = PROXMOX_SETTING.get("domain", DEFAULT_PROXMOX) + PROXMOX_PORT = PROXMOX_SETTING.get("http_port", DEFAULT_PROXMOX_PORT) + PROXMOX_SSL = PROXMOX_SETTING.get("ssl", DEFAULT_PROXMOX_SSL) + + # ACCESS + PROXMOX_USER = PROXMOX_SETTING.get("user", DEFAULT_PROXMOX_USER) + PROXMOX_PASSWORD = PROXMOX_SETTING.get("password", DEFAULT_PROXMOX_PASSWORD) + + PROXMOX_TOKEN = PROXMOX_SETTING.get("token", DEFAULT_PROXMOX_TOKEN) + if PROXMOX_TOKEN != None: + PROXMOX_TOKEN_NAME = PROXMOX_TOKEN.get("name", DEFAULT_PROXMOX_TOKEN_NAME) + PROXMOX_TOKEN_VALUE = PROXMOX_TOKEN.get("value", DEFAULT_PROXMOX_TOKEN_VALUE) + + output = { + 'PROXMOX': PROXMOX, + 'PROXMOX_PORT': PROXMOX_PORT, + 'PROXMOX_SSL': PROXMOX_SSL, + 'PROXMOX_TOKEN': PROXMOX_TOKEN, + 'PROXMOX_TOKEN_NAME': PROXMOX_TOKEN_NAME, + 'PROXMOX_TOKEN_VALUE': PROXMOX_TOKEN_VALUE + } + #################################################################################################### + # # + # WITH PLUGIN CONFIGURED, STARTS BOTH PROXMOX AND NETBOX SESSION # + # # + #################################################################################################### + + # + # PROXMOX SESSION + # + # Check if token was provided + if PROXMOX_TOKEN_VALUE != None and len(PROXMOX_TOKEN_VALUE) > 0: + try: + if PROXMOX_SSL == False: + # DISABLE SSL WARNING + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + # Start PROXMOX session using TOKEN + PROXMOX_SESSION = ProxmoxAPI( + PROXMOX, + port=PROXMOX_PORT, + user=PROXMOX_USER, + token_name=PROXMOX_TOKEN_NAME, + token_value=PROXMOX_TOKEN_VALUE, + verify_ssl=PROXMOX_SSL + ) + output['PROXMOX_SESSION'] = PROXMOX_SESSION + return output + except: + raise RuntimeError(f'Error trying to initialize Proxmox Session using TOKEN (token_name: {PROXMOX_TOKEN_NAME} and token_value: {PROXMOX_TOKEN_VALUE} provided') + return None + + # If token not provided, start session using user and passwd + else: + try: + # Start PROXMOX session using USER CREDENTIALS + PROXMOX_SESSION = ProxmoxAPI( + PROXMOX, + port=PROXMOX_PORT, + user=PROXMOX_USER, + password=PROXMOX_PASSWORD, + verify_ssl=PROXMOX_SSL + ) + output['PROXMOX_SESSION'] = PROXMOX_SESSION + return output + except: + raise RuntimeError(f'Error trying to initialize Proxmox Session using USER {PROXMOX_USER} and PASSWORD') + return None + +for s in PROXMOX_SETTING: + P_Setting = get_proxmox_session(s) + if P_Setting is not None: + v = P_Setting['PROXMOX'] + PROXMOX_SESSIONS[v] = P_Setting # # NETBOX SESSION diff --git a/netbox_proxbox/proxbox_api/remove.py b/netbox_proxbox/proxbox_api/remove.py index 9556d4b..fe7fb2e 100755 --- a/netbox_proxbox/proxbox_api/remove.py +++ b/netbox_proxbox/proxbox_api/remove.py @@ -2,15 +2,18 @@ # PLUGIN_CONFIG variables from .plugins_config import ( - PROXMOX_SESSION as proxmox, NETBOX_SESSION as nb, ) import logging # Verify if VM/CT exists on Proxmox -def is_vm_on_proxmox(netbox_vm): +def is_vm_on_proxmox(proxmox_session, netbox_vm): # Get json of all virtual machines from Proxmox + proxmox = proxmox_session.get('PROXMOX_SESSION') + PROXMOX = proxmox_session.get('PROXMOX') + PROXMOX_PORT = proxmox_session.get('PROXMOX_PORT') + all_proxmox_vms = proxmox.cluster.resources.get(type='vm') # Netbox name @@ -69,7 +72,7 @@ def is_vm_on_proxmox(netbox_vm): # If 'local_context' is null, try updating it to be able to get ID from VM if local_context == None: - local_context_updated = updates.local_context_data(netbox_vm, all_proxmox_vms[name_index]) + local_context_updated = updates.local_context_data(netbox_vm, all_proxmox_vms[name_index], PROXMOX, PROXMOX_PORT) if local_context_updated == True: local_context = netbox_vm.local_context_data @@ -105,7 +108,7 @@ def is_vm_on_proxmox(netbox_vm): # Comparison failed, not able to find VM on Proxmox return False -def all(): +def all(proxmox_session): json_vm_all = [] # Get all VM/CTs from Netbox @@ -120,7 +123,7 @@ def all(): json_vm["name"] = netbox_name # Verify if VM exists on Proxmox - vm_on_proxmox = is_vm_on_proxmox(nb_vm_each) + vm_on_proxmox = is_vm_on_proxmox(proxmox_session, nb_vm_each) if vm_on_proxmox == True: log_message = f'[OK] VM exists on both systems (Netbox and Proxmox) -> {netbox_name}' diff --git a/netbox_proxbox/proxbox_api/update.py b/netbox_proxbox/proxbox_api/update.py index 0b0b5b5..b9086cb 100755 --- a/netbox_proxbox/proxbox_api/update.py +++ b/netbox_proxbox/proxbox_api/update.py @@ -1,13 +1,10 @@ +import time +from threading import Thread # PLUGIN_CONFIG variables from .plugins_config import ( - PROXMOX, - PROXMOX_PORT, - PROXMOX_USER, - PROXMOX_PASSWORD, - PROXMOX_SSL, NETBOX, NETBOX_TOKEN, - PROXMOX_SESSION as proxmox, + PROXMOX_SESSIONS as proxmox_sessions, NETBOX_SESSION as nb, ) @@ -22,7 +19,10 @@ import logging # Call all functions to update Virtual Machine -def vm_full_update(netbox_vm, proxmox_vm): +def vm_full_update(proxmox_session, netbox_vm, proxmox_vm): + proxmox = proxmox_session.get('PROXMOX_SESSION') + PROXMOX = proxmox_session.get('PROXMOX') + PROXMOX_PORT = proxmox_session.get('PROXMOX_PORT') changes = {} # Update 'status' field, if necessary. @@ -32,13 +32,13 @@ def vm_full_update(netbox_vm, proxmox_vm): custom_fields_updated = updates.virtual_machine.custom_fields(netbox_vm, proxmox_vm) # Update 'local_context_data' json, if necessary. - local_context_updated = updates.virtual_machine.local_context_data(netbox_vm, proxmox_vm) + local_context_updated = updates.virtual_machine.local_context_data(netbox_vm, proxmox_vm, PROXMOX, PROXMOX_PORT) # Update 'resources', like CPU, Memory and Disk, if necessary. resources_updated = updates.virtual_machine.resources(netbox_vm, proxmox_vm) - interfaces_updated = updates.virtual_machine.interfaces(netbox_vm, proxmox_vm) - ips_updated = updates.virtual_machine.interfaces_ips(netbox_vm, proxmox_vm) + interfaces_updated = updates.virtual_machine.interfaces(proxmox,netbox_vm, proxmox_vm) + ips_updated = updates.virtual_machine.interfaces_ips(proxmox, netbox_vm, proxmox_vm) tag_updated = updates.extras.tag(netbox_vm) @@ -57,11 +57,11 @@ def vm_full_update(netbox_vm, proxmox_vm): -def node_full_update(netbox_node, proxmox_json, proxmox_cluster): +def node_full_update(proxmox, netbox_node, proxmox_json, proxmox_cluster): changes = {} status_updated = updates.node.status(netbox_node, proxmox_json) - cluster_updated = updates.node.cluster(netbox_node, proxmox_json, proxmox_cluster) + cluster_updated = updates.node.cluster(proxmox, netbox_node, proxmox_json, proxmox_cluster) interfaces_updated = updates.node.interfaces(netbox_node, proxmox_json) changes = { @@ -73,7 +73,7 @@ def node_full_update(netbox_node, proxmox_json, proxmox_cluster): return changes -def search_by_proxmox_id(proxmox_id): +def search_by_proxmox_id(proxmox, proxmox_id): all_proxmox_vms = proxmox.cluster.resources.get(type='vm') for px_vm in all_proxmox_vms: @@ -88,7 +88,7 @@ def search_by_proxmox_id(proxmox_id): -def search_by_proxmox_name(proxmox_name): +def search_by_proxmox_name(proxmox, proxmox_name): all_proxmox_vms = proxmox.cluster.resources.get(type='vm') for px_vm in all_proxmox_vms: @@ -104,7 +104,7 @@ def search_by_proxmox_name(proxmox_name): def search_by_id(id): - # Save Netbox VirtualMachine object + # Get Netbox VirtualMachine object netbox_obj = nb.virtualization.virtual_machines.get(id) proxmox_name = netbox_obj.name @@ -127,6 +127,9 @@ def search_by_id(id): # Makes all necessary checks so that VM/CT exist on Netbox. def virtual_machine(**kwargs): + print('[OK] STARTING PROCESS FOR VIRTUAL MACHINE') + proxmox_session = kwargs.get('proxmox_session') + proxmox = proxmox_session.get('PROXMOX_SESSION') # JSON containing the result of the VM changes json_vm = {} @@ -203,7 +206,7 @@ def virtual_machine(**kwargs): # Search using Proxmox ID if 'int' in str(search_result_type): - proxmox_json = search_by_proxmox_id(search_result) + proxmox_json = search_by_proxmox_id(proxmox, search_result) # Analyze search result and returns error, if null value. if proxmox_json == None: @@ -215,7 +218,7 @@ def virtual_machine(**kwargs): # Search using Proxmox NAME elif 'str' in str(search_result_type): - proxmox_json = search_by_proxmox_name(search_result) + proxmox_json = search_by_proxmox_name(proxmox, search_result) # Analyze search result and returns error, if null value. if proxmox_json == None: @@ -228,7 +231,7 @@ def virtual_machine(**kwargs): else: # Search VM JSON of Proxmox using argument 'proxmox_id' if proxmox_id != None: - proxmox_json = search_by_proxmox_id(proxmox_id) + proxmox_json = search_by_proxmox_id(proxmox, proxmox_id) # Analyze search result and returns error, if null value. if proxmox_json == None: @@ -241,7 +244,7 @@ def virtual_machine(**kwargs): else: # Search using Proxmox NAME if name != None: - proxmox_json = search_by_proxmox_name(name) + proxmox_json = search_by_proxmox_name(proxmox, name) # Analyze search result and returns error, if null value. if proxmox_json == None: @@ -255,8 +258,30 @@ def virtual_machine(**kwargs): return False # Search Netbox object by name gotten from Proxmox - netbox_vm = nb.virtualization.virtual_machines.get(name = proxmox_vm_name) + # netbox_vm = nb.virtualization.virtual_machines.get(name = proxmox_vm_name) + print('[OK] Getting node') + node = proxmox_json['node'] + cluster = None + try: + cluster = kwargs.get('cluster') + print('[OK] Getting cluster') + except Exception as e: + cluster = None + if cluster is None: + cluster = create.virtualization.cluster(proxmox) + + print('[OK] Getting vmid') + vmid = proxmox_json['vmid'] + print('[OK] Getting getting vm from netbox for {} , {}, {}'.format(cluster.name, vmid, node)) + netbox_vm_all = nb.virtualization.virtual_machines.filter(cluster=cluster.name, cf_proxmox_id=vmid, + cf_proxmox_node=node) + + netbox_vm = None + for vm in netbox_vm_all: + netbox_vm = vm + break # ???? + duplicate = False try: # Check if Proxbox tag exist. @@ -270,7 +295,7 @@ def virtual_machine(**kwargs): # VM Found: if netbox_vm: # Update Netbox information - full_update = vm_full_update(netbox_vm, proxmox_json) + full_update = vm_full_update(proxmox_session, netbox_vm, proxmox_json) # I made this way since dict.update didn't work json_vm["vm_id"] = netbox_vm.id @@ -298,12 +323,12 @@ def virtual_machine(**kwargs): logging.info(f'[OK] VM does not exist on Netbox -> {proxmox_vm_name}') # Analyze if VM was sucessfully created. - netbox_vm = create.virtualization.virtual_machine(proxmox_json, duplicate) + netbox_vm = create.virtualization.virtual_machine(proxmox, proxmox_json, duplicate) # VM created with basic information if netbox_vm != None: # Update rest of configuration - full_update = vm_full_update(netbox_vm, proxmox_json) + full_update = vm_full_update(proxmox_session, netbox_vm, proxmox_json) json_vm = full_update json_vm["vm_id"] = netbox_vm.id @@ -329,6 +354,7 @@ def virtual_machine(**kwargs): # VM not created json_vm["result"] = False + print('[OK] FINISH PROCESS FOR VIRTUAL MACHINE') return json_vm @@ -336,12 +362,13 @@ def virtual_machine(**kwargs): def nodes(**kwargs): proxmox_cluster = kwargs.get('proxmox_cluster') proxmox_json = kwargs.get('proxmox_json') + proxmox = kwargs.get('proxmox') proxmox_node_name = proxmox_json.get("name") def create_node(): # If node does not exist, create it. - netbox_node = create.dcim.node(proxmox_json) + netbox_node = create.dcim.node(proxmox, proxmox_json) # Node created if netbox_node != None: @@ -398,7 +425,7 @@ def create_node(): netbox_node = netbox_search # Update Netbox node information, if necessary. - full_update = node_full_update(netbox_node, proxmox_json, proxmox_cluster) + full_update = node_full_update(proxmox, netbox_node, proxmox_json, proxmox_cluster) json_node = full_update full_update_list = list(full_update.values()) @@ -427,17 +454,68 @@ def create_node(): return json_node - +def run_process_in_thread(proxmox_session, key, result, index, **kwargs): + output = {} + try: + print("Processing data for: {0}".format(key)) + output = process_all_in_session(proxmox_session, **kwargs) + result[index] = output + except Exception as e: + message = "OS error: {0}".format(e) + print(message) + result[index] = { + 'message': message + } + return output # Makes everything needed so that VIRTUAL MACHINES / CONTAINERS, NODES and CLUSTER exists on Netbox def all(**kwargs): + run_with_threads = kwargs.get("run_with_threads", True) + start_time = time.time() + result = [] + if run_with_threads: + print("Start process with threading") + threads = [None] * len(proxmox_sessions) + results = [None] * len(proxmox_sessions) + i = 0 + for key in proxmox_sessions: + proxmox_session = proxmox_sessions[key] + threads[i] = Thread(target=run_process_in_thread, args=(proxmox_session, key, results, i), kwargs=kwargs) + threads[i].start() + i = i + 1 + + p = len(threads) + r = range(p) + for i in r: + threads[(p - 1) - i].join() + else: + print("Start process Sequential") + for key in proxmox_sessions: + try: + session = proxmox_sessions[key] + print("Processing data for: {0}".format(key)) + output = process_all_in_session(session, **kwargs) + result.append(output) + except Exception as e: + message = "OS error: {0}".format(e) + print(message) + output = { + 'message': message + } + result.append(output) + + print("--- %s seconds ---" % (time.time() - start_time)) + return results + +def process_all_in_session(proxmox_session, **kwargs): print("[Proxbox - Netbox plugin | Update All]") + proxmox = proxmox_session.get('PROXMOX_SESSION') cluster_all = proxmox.cluster.status.get() # # CLUSTER # - cluster = create.virtualization.cluster() + cluster = create.virtualization.cluster(proxmox) print('\n\n\nCLUSTER...') try: @@ -455,8 +533,7 @@ def all(**kwargs): # Get all NODES from Proxmox for px_node_each in proxmox_nodes: - node_updated = nodes(proxmox_json = px_node_each, proxmox_cluster = proxmox_cluster) - + node_updated = nodes(proxmox_json = px_node_each, proxmox_cluster = proxmox_cluster, proxmox = proxmox) nodes_list.append(node_updated) @@ -468,9 +545,9 @@ def all(**kwargs): print('\nUPDATE ALL...') # Get all VM/CTs from Proxmox - for px_vm_each in proxmox.cluster.resources.get(type='vm'): - vm_updated = virtual_machine(proxmox_json = px_vm_each) - + node_vms_all = proxmox.cluster.resources.get(type='vm') + for px_vm_each in node_vms_all: + vm_updated = virtual_machine(proxmox_json = px_vm_each, proxmox_session = proxmox_session, cluster = cluster) virtualmachines_list.append(vm_updated) # Get "remove_unused" passed on function call @@ -479,7 +556,7 @@ def all(**kwargs): # Remove Netbox's old data if remove_unused == True: print('\nREMOVE UNUSED DATA...') - remove_info = remove.all() + remove_info = remove.all(proxmox_session) else: remove_info = False # diff --git a/netbox_proxbox/proxbox_api/updates/node.py b/netbox_proxbox/proxbox_api/updates/node.py index c8f5f8c..3a6c340 100755 --- a/netbox_proxbox/proxbox_api/updates/node.py +++ b/netbox_proxbox/proxbox_api/updates/node.py @@ -1,12 +1,6 @@ from ..plugins_config import ( - PROXMOX, - PROXMOX_PORT, - PROXMOX_USER, - PROXMOX_PASSWORD, - PROXMOX_SSL, NETBOX, NETBOX_TOKEN, - PROXMOX_SESSION as proxmox, NETBOX_SESSION as nb, ) @@ -57,7 +51,7 @@ def status(netbox_node, proxmox_node): # Update CLUSTER field on /dcim/device/{id} -def cluster(netbox_node, proxmox_node, proxmox_cluster): +def cluster(proxmox, netbox_node, proxmox_node, proxmox_cluster): # # Compare CLUSTER # @@ -69,7 +63,7 @@ def cluster(netbox_node, proxmox_node, proxmox_cluster): # If cluster is not filled or even filled, but different from actual cluster, update it. if netbox_node.cluster.name != proxmox_cluster['name'] or netbox_node.cluster.name == None: # Search for Proxmox Cluster using create.cluster() function - cluster_id = create.virtualization.cluster().id + cluster_id = create.virtualization.cluster(proxmox).id # Use Cluster ID to update NODE information netbox_node.cluster.id = cluster_id @@ -106,7 +100,7 @@ def cluster(netbox_node, proxmox_node, proxmox_cluster): return cluster_updated -def interfaces(netbox_node, proxmox_json): +def interfaces(proxmox, netbox_node, proxmox_json): updated = False _int_port = ['OVSIntPort'] _lag_port = ['OVSBond'] diff --git a/netbox_proxbox/proxbox_api/updates/virtual_machine.py b/netbox_proxbox/proxbox_api/updates/virtual_machine.py index c1dc232..9ad2273 100755 --- a/netbox_proxbox/proxbox_api/updates/virtual_machine.py +++ b/netbox_proxbox/proxbox_api/updates/virtual_machine.py @@ -4,14 +4,8 @@ # PLUGIN_CONFIG variables from ..plugins_config import ( - PROXMOX, - PROXMOX_PORT, - PROXMOX_USER, - PROXMOX_PASSWORD, - PROXMOX_SSL, NETBOX, NETBOX_TOKEN, - PROXMOX_SESSION as proxmox, NETBOX_SESSION as nb, ) @@ -177,7 +171,7 @@ def custom_fields(netbox_vm, proxmox_vm): # Update 'local_context_data' field on Netbox Virtual Machine based on Proxbox -def local_context_data(netbox_vm, proxmox_vm): +def local_context_data(netbox_vm, proxmox_vm, PROXMOX, PROXMOX_PORT): current_local_context = netbox_vm.local_context_data proxmox_values = {} @@ -219,12 +213,6 @@ def local_context_data(netbox_vm, proxmox_vm): return False - - - - - - # Updates following fields based on Proxmox: "vcpus", "memory", "disk", if necessary. def resources(netbox_vm, proxmox_vm): # Save values from Proxmox @@ -285,7 +273,7 @@ def resources(netbox_vm, proxmox_vm): else: return False -def interfaces(netbox_vm, proxmox_vm): +def interfaces(proxmox, netbox_vm, proxmox_vm): updated = False try: if proxmox_vm['type'] == 'qemu': @@ -369,7 +357,7 @@ def interfaces(netbox_vm, proxmox_vm): return False return updated -def interfaces_ips(netbox_vm, proxmox_vm): +def interfaces_ips(proxmox, netbox_vm, proxmox_vm): updated = False if proxmox_vm['status'] == 'running': _pmx_ips = [] From 5310468b2cb4d40cd808a8f897d2ab872f74e202 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sat, 8 Jul 2023 10:44:42 +0200 Subject: [PATCH 2/7] feat(Multi server): Update missing components https://github.com/netdevopsbr/netbox-proxbox/issues/33 Signed-off-by: devopstales <42894256+devopstales@users.noreply.github.com> --- netbox_proxbox/proxbox_api/plugins_config.py | 14 ++++++-------- netbox_proxbox/proxbox_api/remove.py | 6 +++--- netbox_proxbox/proxbox_api/update.py | 9 ++++++--- .../proxbox_api/updates/virtual_machine.py | 2 +- reinstall.sh | 9 +++++++++ 5 files changed, 25 insertions(+), 15 deletions(-) create mode 100755 reinstall.sh diff --git a/netbox_proxbox/proxbox_api/plugins_config.py b/netbox_proxbox/proxbox_api/plugins_config.py index cf1eef7..df7e70d 100755 --- a/netbox_proxbox/proxbox_api/plugins_config.py +++ b/netbox_proxbox/proxbox_api/plugins_config.py @@ -33,15 +33,15 @@ # Proxmox related settings # # API URI -DEFAULT_PROXMOX = DEFAULT_PROXMOX_SETTING.get("domain") -DEFAULT_PROXMOX_PORT = DEFAULT_PROXMOX_SETTING.get("http_port") -DEFAULT_PROXMOX_SSL = DEFAULT_PROXMOX_SETTING.get("ssl") +DEFAULT_PROXMOX = DEFAULT_PROXMOX_SETTING[0].get("domain") +DEFAULT_PROXMOX_PORT = DEFAULT_PROXMOX_SETTING[0].get("http_port") +DEFAULT_PROXMOX_SSL = DEFAULT_PROXMOX_SETTING[0].get("ssl") # ACCESS -DEFAULT_PROXMOX_USER = DEFAULT_PROXMOX_SETTING.get("user") -DEFAULT_PROXMOX_PASSWORD = DEFAULT_PROXMOX_SETTING.get("password") +DEFAULT_PROXMOX_USER = DEFAULT_PROXMOX_SETTING[0].get("user") +DEFAULT_PROXMOX_PASSWORD = DEFAULT_PROXMOX_SETTING[0].get("password") -DEFAULT_PROXMOX_TOKEN = DEFAULT_PROXMOX_SETTING.get("token") +DEFAULT_PROXMOX_TOKEN = DEFAULT_PROXMOX_SETTING[0].get("token") DEFAULT_PROXMOX_TOKEN_NAME = DEFAULT_PROXMOX_TOKEN.get("name", None) DEFAULT_PROXMOX_TOKEN_VALUE = DEFAULT_PROXMOX_TOKEN.get("value", None) @@ -152,7 +152,6 @@ def get_proxmox_session(PROXMOX_SETTING): return output except: raise RuntimeError(f'Error trying to initialize Proxmox Session using TOKEN (token_name: {PROXMOX_TOKEN_NAME} and token_value: {PROXMOX_TOKEN_VALUE} provided') - return None # If token not provided, start session using user and passwd else: @@ -169,7 +168,6 @@ def get_proxmox_session(PROXMOX_SETTING): return output except: raise RuntimeError(f'Error trying to initialize Proxmox Session using USER {PROXMOX_USER} and PASSWORD') - return None for s in PROXMOX_SETTING: P_Setting = get_proxmox_session(s) diff --git a/netbox_proxbox/proxbox_api/remove.py b/netbox_proxbox/proxbox_api/remove.py index fe7fb2e..a87dca0 100755 --- a/netbox_proxbox/proxbox_api/remove.py +++ b/netbox_proxbox/proxbox_api/remove.py @@ -108,11 +108,11 @@ def is_vm_on_proxmox(proxmox_session, netbox_vm): # Comparison failed, not able to find VM on Proxmox return False -def all(proxmox_session): +def all(proxmox_session, cluster): json_vm_all = [] - # Get all VM/CTs from Netbox - netbox_all_vms = nb.virtualization.virtual_machines.all() + # Get VM/CTs of the specific cluster from Netbox + netbox_all_vms = nb.virtualization.virtual_machines.filter(cluster=cluster.name) for nb_vm_each in netbox_all_vms: json_vm = {} diff --git a/netbox_proxbox/proxbox_api/update.py b/netbox_proxbox/proxbox_api/update.py index b9086cb..5270fd9 100755 --- a/netbox_proxbox/proxbox_api/update.py +++ b/netbox_proxbox/proxbox_api/update.py @@ -37,7 +37,7 @@ def vm_full_update(proxmox_session, netbox_vm, proxmox_vm): # Update 'resources', like CPU, Memory and Disk, if necessary. resources_updated = updates.virtual_machine.resources(netbox_vm, proxmox_vm) - interfaces_updated = updates.virtual_machine.interfaces(proxmox,netbox_vm, proxmox_vm) + interfaces_updated = updates.virtual_machine.interfaces(proxmox, netbox_vm, proxmox_vm) ips_updated = updates.virtual_machine.interfaces_ips(proxmox, netbox_vm, proxmox_vm) tag_updated = updates.extras.tag(netbox_vm) @@ -62,7 +62,7 @@ def node_full_update(proxmox, netbox_node, proxmox_json, proxmox_cluster): status_updated = updates.node.status(netbox_node, proxmox_json) cluster_updated = updates.node.cluster(proxmox, netbox_node, proxmox_json, proxmox_cluster) - interfaces_updated = updates.node.interfaces(netbox_node, proxmox_json) + interfaces_updated = updates.node.interfaces(proxmox, netbox_node, proxmox_json) changes = { "status" : status_updated, @@ -394,6 +394,9 @@ def create_node(): else: logging.error(f'[ERROR] Something went wrong when creating the node.-> {proxmox_node_name} (2)') json_node = {} + json_node["status"] = None + json_node["cluster"] = None + json_node["interfaces"] = None json_node["result"] = False return json_node @@ -556,7 +559,7 @@ def process_all_in_session(proxmox_session, **kwargs): # Remove Netbox's old data if remove_unused == True: print('\nREMOVE UNUSED DATA...') - remove_info = remove.all(proxmox_session) + remove_info = remove.all(proxmox_session, cluster) else: remove_info = False # diff --git a/netbox_proxbox/proxbox_api/updates/virtual_machine.py b/netbox_proxbox/proxbox_api/updates/virtual_machine.py index 9ad2273..575bdc0 100755 --- a/netbox_proxbox/proxbox_api/updates/virtual_machine.py +++ b/netbox_proxbox/proxbox_api/updates/virtual_machine.py @@ -78,7 +78,7 @@ def http_update_custom_fields(**kwargs): # HTTP PATCH Request (partially update) # # URL - url = '{}/api/virtualization/virtual-machines/{}/'.format(domain_with_http, vm_id) + url = '{}api/virtualization/virtual-machines/{}/'.format(domain_with_http, vm_id) # HTTP Request Headers headers = { diff --git a/reinstall.sh b/reinstall.sh new file mode 100755 index 0000000..9e51666 --- /dev/null +++ b/reinstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +systemctl stop netbox.service + +pip3 uninstall netbox-proxbox -y + +python3 setup.py develop + +systemctl start netbox.service From 8edcf6584db764ea4e005e9179878c134d4b5235 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sat, 8 Jul 2023 10:48:26 +0200 Subject: [PATCH 3/7] feat(Multi server): Update README and add custom fields csv https://github.com/netdevopsbr/netbox-proxbox/issues/33 Signed-off-by: devopstales <42894256+devopstales@users.noreply.github.com> --- README.md | 27 +++++++++++++++------------ netbox_custom_fields.csv | 5 +++++ 2 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 netbox_custom_fields.csv diff --git a/README.md b/README.md index f0cec9d..75e8d30 100755 --- a/README.md +++ b/README.md @@ -153,17 +153,19 @@ Replace the values with your own following the [Configuration Parameters](#2-con ```python PLUGINS_CONFIG = { 'netbox_proxbox': { - 'proxmox': { - 'domain': 'proxbox.example.com', # May also be IP address - 'http_port': 8006, - 'user': 'root@pam', # always required - 'password': 'Strong@P4ssword', # only required, if you don't want to use token based authentication - 'token': { - 'name': 'tokenID', # Only type the token name and not the 'user@pam:tokenID' format - 'value': '039az154-23b2-4be0-8d20-b66abc8c4686' - }, - 'ssl': False - }, + 'proxmox': [ + { + 'domain': 'proxbox.example.com', # May also be IP address + 'http_port': 8006, + 'user': 'root@pam', # always required + 'password': 'Strong@P4ssword', # only required, if you don't want to use token based authentication + 'token': { + 'name': 'tokenID', # Only type the token name and not the 'user@pam:tokenID' format + 'value': '039az154-23b2-4be0-8d20-b66abc8c4686' + }, + 'ssl': False + } + ], 'netbox': { 'domain': 'localhost', # Ensure localhost is added to ALLOWED_HOSTS 'http_port': 8001, # Gunicorn port. @@ -194,6 +196,7 @@ PLUGINS_CONFIG = { ``` (venv) $ cd /opt/netbox/netbox/ (venv) $ python3 manage.py migrate +(venv) $ python3 manage.py collectstatic --no-input ``` --- @@ -211,7 +214,7 @@ Restart the WSGI service to load the new plugin: The following options are available: -* `proxmox`: (Dict) Proxmox related configuration to use proxmoxer. +* `proxmox`: (List) Proxmox related configuration to use proxmoxer. * `proxmox.domain`: (String) Domain or IP address of Proxmox. * `proxmox.http_port`: (Integer) Proxmox HTTP port (default: 8006). * `proxmox.user`: (String) Proxmox Username. diff --git a/netbox_custom_fields.csv b/netbox_custom_fields.csv new file mode 100644 index 0000000..5d69a33 --- /dev/null +++ b/netbox_custom_fields.csv @@ -0,0 +1,5 @@ +Name,Content types,Label,Group name,Type,Required,Description,ID,Default,Search weight,Filter logic,UI visibility,Cloneable,Display weight,Choices,Created,Last updated +proxmox_id,virtualization.virtualmachine,[Proxmox] ID,,Integer,False,Proxmox VM/CT ID,1,,1000,Loose,Read/Write,False,100,[],2023-07-06 12:49,2023-07-06 12:50 +proxmox_keep_interface,dcim.interface,,,Boolean (true/false),False,,4,,1000,Loose,Read/Write,False,100,[],2023-07-06 12:53,2023-07-06 12:53 +proxmox_node,virtualization.virtualmachine,[Proxmox] Node,,Text,False,Proxmox Node (Server),2,,1000,Loose,Read/Write,False,100,[],2023-07-06 12:51,2023-07-06 12:51 +proxmox_type,virtualization.virtualmachine,[Proxmox] Type,,Selection,False,Proxmox type (VM or CT),3,,1000,Loose,Read/Write,False,100,"['qemu', 'lxc']",2023-07-06 12:52,2023-07-06 12:52 From 0b39b777153ff4c312d327bfe89ae50ffd7e2419 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sat, 8 Jul 2023 10:48:52 +0200 Subject: [PATCH 4/7] feat(Multi server): Update UI https://github.com/netdevopsbr/netbox-proxbox/issues/33 Signed-off-by: devopstales <42894256+devopstales@users.noreply.github.com> --- .../templates/netbox_proxbox/home.html | 46 +-- .../proxmox_vm_full_update.html | 344 +++++++++--------- 2 files changed, 197 insertions(+), 193 deletions(-) diff --git a/netbox_proxbox/templates/netbox_proxbox/home.html b/netbox_proxbox/templates/netbox_proxbox/home.html index 8e95eb1..1c64c40 100644 --- a/netbox_proxbox/templates/netbox_proxbox/home.html +++ b/netbox_proxbox/templates/netbox_proxbox/home.html @@ -19,11 +19,12 @@

+ {% for px in configuration.netbox_proxbox.proxmox %}

- + Proxmox Logo
@@ -31,54 +32,54 @@

- {% if configuration.netbox_proxbox.proxmox.domain %} - + {% if px.domain %} + {% else %} {% endif %} - {% if configuration.netbox_proxbox.proxmox.http_port %} - + {% if px.http_port %} + {% else %} {% endif %} - {% if configuration.netbox_proxbox.proxmox.user %} - + {% if px.user %} + {% else %} {% endif %} - {% if configuration.netbox_proxbox.proxmox.password %} + {% if px.password %} {% else %} {% endif %} - {% if configuration.netbox_proxbox.proxmox.token.name %} - + {% if px.token.name %} + {% else %} {% endif %} - {% if configuration.netbox_proxbox.proxmox.token.value %} - + {% if px.token.value %} + {% else %} {% endif %} - {% if configuration.netbox_proxbox.proxmox.ssl %} - + {% if px.ssl %} + {% else %} {% endif %} @@ -96,6 +97,7 @@

+ {% endfor %}
@@ -108,32 +110,32 @@

Domain / IP{{ configuration.netbox_proxbox.proxmox.domain }}{{ px.domain }}{{ default_config.proxmox.domain }} (default)
HTTP Port{{ configuration.netbox_proxbox.proxmox.http_port }}{{ px.http_port }}{{ default_config.proxmox.http_port }} (default)
Proxmox User{{ configuration.netbox_proxbox.proxmox.user }}{{ px.user }}{{ default_config.proxmox.user }} (default)
Proxmox Passwordpassword defined in configuration.py{{ default_config.proxmox.password }} (default)
Token Name{{ configuration.netbox_proxbox.proxmox.token.name }}{{ px.token.name }}{{ default_config.proxmox.token.name }} (default)
Token Value{{ configuration.netbox_proxbox.proxmox.token.value }}{{ px.token.value }}{{ default_config.proxmox.token.value }} (default)
SSL{{ configuration.netbox_proxbox.proxmox.ssl }}{{ px.ssl }}{{ default_config.proxmox.ssl }} (default)
- {% if configuration.netbox_proxbox.netbox.domain %} - + {% if px.netbox.domain %} + {% else %} {% endif %} - {% if configuration.netbox_proxbox.netbox.http_port %} - + {% if px.netbox.http_port %} + {% else %} {% endif %} - {% if configuration.netbox_proxbox.netbox.token %} - + {% if px.netbox.token %} + {% else %} {% endif %} - {% if configuration.netbox_proxbox.netbox.ssl %} - + {% if px.netbox.ssl %} + {% else %} {% endif %} diff --git a/netbox_proxbox/templates/netbox_proxbox/proxmox_vm_full_update.html b/netbox_proxbox/templates/netbox_proxbox/proxmox_vm_full_update.html index 21a8cb1..616073f 100755 --- a/netbox_proxbox/templates/netbox_proxbox/proxmox_vm_full_update.html +++ b/netbox_proxbox/templates/netbox_proxbox/proxmox_vm_full_update.html @@ -7,184 +7,186 @@

Virtual Machines and Containers

- {% if proxmox.nodes %} -

Nodes

-
Domain / IP{{ configuration.netbox_proxbox.netbox.domain }}{{ px.netbox.domain }}{{ default_config.netbox.domain }} (default)
HTTP Port{{ configuration.netbox_proxbox.netbox.http_port }}{{ px.netbox.http_port }}{{ default_config.netbox.http_port }} (default)
Token{{ configuration.netbox_proxbox.netbox.token }}{{ px.netbox.token }}{{ default_config.netbox.token }} (default)
SSL{{ configuration.netbox_proxbox.netbox.ssl }}{{ px.netbox.ssl }}{{ default_config.netbox.ssl }} (default)
- + {% for px in proxmox %} + {% if px.nodes %} +

Nodes

+
+ + + + + + + + + + + {% for node in px.nodes %} - - - - - - - - - {% for node in proxmox.nodes %} - - + - - - - - {% endfor %} - -
Node NameNode status changedCluster changedInterface changedUpdate result
Node NameNode status changedCluster changedInterface changedUpdate result
- - {% if node.node_id %} - + + + {% if node.node_id %} + + {{ node.name }} + + {% else %} {{ node.name }} - + {% endif %} + + + {% if node.status %} + + {{ node.status }} + {% else %} - {{ node.name }} + + {{ node.status }} + {% endif %} - - - {% if node.status %} - - {{ node.status }} - - {% else %} - - {{ node.status }} - - {% endif %} - - {% if node.cluster %} - - {{ node.cluster }} - - {% else %} - - {{ node.cluster }} - - {% endif %} - - {% if node.interfaces %} - - {{ node.interfaces }} - - {% else %} - - {{ node.interfaces }} - - {% endif %} - - {% if node.result %} - - {{ node.result }} - - {% else %} - - {{ node.result }} - - {% endif %} -
- -

- {% endif %} - {% if proxmox.virtualmachines %} -

Virtual Machines and Containers

- - - - - - - - - - - + + + - - - {% for virtualmachine in proxmox.virtualmachines %} - - +
VM/CT NameStatus changedCustom FieldsLocal contextResourcesTagUpdate result + {% if node.cluster %} + + {{ node.cluster }} + + {% else %} + + {{ node.cluster }} + + {% endif %} + + {% if node.interfaces %} + + {{ node.interfaces }} + + {% else %} + + {{ node.interfaces }} + + {% endif %} + + {% if node.result %} + + {{ node.result }} + + {% else %} + + {{ node.result }} + + {% endif %} +
- - {% if virtualmachine.vm_id %} - + {% endfor %} +
+ +

+ {% endif %} + {% if px.virtualmachines %} +

Virtual Machines and Containers

+ + + + + + + + + + + + + + + {% for virtualmachine in px.virtualmachines %} + + + - - - - - - - - {% endfor %} - -
VM/CT NameStatus changedCustom FieldsLocal contextResourcesTagUpdate result
+ + {% if virtualmachine.vm_id %} + + {{ virtualmachine.name }} + + {% else %} {{ virtualmachine.name }} - + {% endif %} + + + {% if virtualmachine.status %} + + {{ virtualmachine.status }} + {% else %} - {{ virtualmachine.name }} - {% endif %} - - - {% if virtualmachine.status %} - - {{ virtualmachine.status }} - - {% else %} - - {{ virtualmachine.status }} - - {% endif %} - - {% if virtualmachine.custom_fields %} - - {{ virtualmachine.custom_fields }} - - {% else %} - - {{ virtualmachine.custom_fields }} - - {% endif %} - - {% if virtualmachine.local_context %} - - {{ virtualmachine.local_context }} - - {% else %} - - {{ virtualmachine.local_context }} - - {% endif %} - - {% if virtualmachine.resources %} - - {{ virtualmachine.resources }} - - {% else %} - - {{ virtualmachine.resources }} - - {% endif %} - - {% if virtualmachine.tag %} - - {{ virtualmachine.tag }} - - {% else %} - - {{ virtualmachine.tag }} - - {% endif %} - - {% if virtualmachine.result %} - - {{ virtualmachine.result }} - - {% else %} - - {{ virtualmachine.result }} - - {% endif %} -
- -

- {% endif %} + + {{ virtualmachine.status }} + + {% endif %} + + + {% if virtualmachine.custom_fields %} + + {{ virtualmachine.custom_fields }} + + {% else %} + + {{ virtualmachine.custom_fields }} + + {% endif %} + + + {% if virtualmachine.local_context %} + + {{ virtualmachine.local_context }} + + {% else %} + + {{ virtualmachine.local_context }} + + {% endif %} + + + {% if virtualmachine.resources %} + + {{ virtualmachine.resources }} + + {% else %} + + {{ virtualmachine.resources }} + + {% endif %} + + + {% if virtualmachine.tag %} + + {{ virtualmachine.tag }} + + {% else %} + + {{ virtualmachine.tag }} + + {% endif %} + + + {% if virtualmachine.result %} + + {{ virtualmachine.result }} + + {% else %} + + {{ virtualmachine.result }} + + {% endif %} + + + {% endfor %} + + + +

+ {% endif %} + {% endfor %}
From 37fbd62cd9bc9404b8e2360fcd7d968b3ce9f565 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sat, 8 Jul 2023 10:50:55 +0200 Subject: [PATCH 5/7] fix: Create non existing vlans https://github.com/netdevopsbr/netbox-proxbox/issues/110 Signed-off-by: devopstales <42894256+devopstales@users.noreply.github.com> --- netbox_proxbox/proxbox_api/updates/node.py | 38 +++++++++++++++++----- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/netbox_proxbox/proxbox_api/updates/node.py b/netbox_proxbox/proxbox_api/updates/node.py index 3a6c340..ca08318 100755 --- a/netbox_proxbox/proxbox_api/updates/node.py +++ b/netbox_proxbox/proxbox_api/updates/node.py @@ -112,6 +112,20 @@ def interfaces(proxmox, netbox_node, proxmox_json): _bond = [iface for iface in proxmox.nodes(proxmox_json['name']).network.get() if iface['type'] in _lag_port] _bridge = [iface for iface in proxmox.nodes(proxmox_json['name']).network.get() if iface['type'] in _brg_port] + def get_or_create_vlan(vlan_id): + # FIXME: we may need to specify the datacenter + # since users may have same vlan id in multiple dc + vlan = nb.ipam.vlans.get( + vid=vlan_id, + ) + if vlan is None: + vlan = nb.ipam.vlans.create( + name='VLAN {}'.format(vlan_id), + vid=vlan_id, + group=None, + ) + return vlan + for iface in _eth: ntb_iface = list(nb.dcim.interfaces.filter(device_id=netbox_node.id, name=iface['iface'])) if iface.get('ovs_tag') is not None: @@ -122,7 +136,8 @@ def interfaces(proxmox, netbox_node, proxmox_json): pmx_if = next((_if for _if in _pmx_iface if _if['name'] == iface['iface']), None) if not len(ntb_iface): if len(_tagged_vlans): - ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=pmx_if['name'], type='other', mtu=pmx_if['mtu'], mode='tagged', tagged_vlans=[nb.ipam.vlans.get(vid=_tagged_vlans[0]).id]) + _nb_vlan = get_or_create_vlan(_tagged_vlans[0]) + ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=pmx_if['name'], type='other', mtu=pmx_if['mtu'], mode='tagged', tagged_vlans=[_nb_vlan.id]) else: ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=pmx_if['name'], type='other', mtu=pmx_if['mtu']) updated = True @@ -132,7 +147,8 @@ def interfaces(proxmox, netbox_node, proxmox_json): ntb_if = next((_if for _if in _ntb_iface if _if['name'] == iface['iface']), None) if pmx_if != ntb_if: if len(pmx_if['tagged_vlans']): - nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu'], 'mode': 'tagged', 'tagged_vlans': [nb.ipam.vlans.get(vid=_tagged_vlans[0]).id]}]) + _nb_vlan = get_or_create_vlan(_tagged_vlans[0]) + nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu'], 'mode': 'tagged', 'tagged_vlans': [_nb_vlan.id]}]) else: nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu']}]) updated = True @@ -146,7 +162,8 @@ def interfaces(proxmox, netbox_node, proxmox_json): _pmx_iface.append({'name': iface['iface'], 'mtu' : int(iface.get('mtu', 1500)), 'tagged_vlans': _tagged_vlans}) if not len(ntb_iface): if len(_tagged_vlans): - ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='lag', mtu=int(iface.get('mtu', 1500)), mode='tagged', tagged_vlans=[nb.ipam.vlans.get(vid=_tagged_vlans[0]).id]) + _nb_vlan = get_or_create_vlan(_tagged_vlans[0]) + ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='lag', mtu=int(iface.get('mtu', 1500)), mode='tagged', tagged_vlans=[_nb_vlan.id]) else: ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='lag', mtu=int(iface.get('mtu', 1500))) else: @@ -156,7 +173,8 @@ def interfaces(proxmox, netbox_node, proxmox_json): ntb_if = next((_if for _if in _ntb_iface if _if['name'] == iface['iface']), None) if pmx_if != ntb_if: if len(pmx_if['tagged_vlans']): - nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu'], 'mode': 'tagged', 'tagged_vlans': [nb.ipam.vlans.get(vid=_tagged_vlans[0]).id]}]) + _nb_vlan = get_or_create_vlan(_tagged_vlans[0]) + nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu'], 'mode': 'tagged', 'tagged_vlans': [_nb_vlan.id]}]) else: nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu']}]) updated = True @@ -175,7 +193,8 @@ def interfaces(proxmox, netbox_node, proxmox_json): _pmx_iface.append({'name': iface['iface'], 'mtu' : int(iface.get('mtu', 1500)), 'tagged_vlans': _tagged_vlans}) if not len(ntb_iface): if len(_tagged_vlans): - ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='virtual', mtu=int(iface.get('mtu', 1500)), mode='tagged', tagged_vlans=[nb.ipam.vlans.get(vid=_tagged_vlans[0]).id]) + _nb_vlan = get_or_create_vlan(_tagged_vlans[0]) + ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='virtual', mtu=int(iface.get('mtu', 1500)), mode='tagged', tagged_vlans=[_nb_vlan.id]) else: ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='virtual', mtu=int(iface.get('mtu', 1500))) else: @@ -185,7 +204,8 @@ def interfaces(proxmox, netbox_node, proxmox_json): ntb_if = next((_if for _if in _ntb_iface if _if['name'] == iface['iface']), None) if pmx_if != ntb_if: if len(pmx_if['tagged_vlans']): - nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu'], 'mode': 'tagged', 'tagged_vlans': [nb.ipam.vlans.get(vid=_tagged_vlans[0]).id]}]) + _nb_vlan = get_or_create_vlan(_tagged_vlans[0]) + nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu'], 'mode': 'tagged', 'tagged_vlans': [_nb_vlan.id]}]) else: nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu']}]) updated = True @@ -199,7 +219,8 @@ def interfaces(proxmox, netbox_node, proxmox_json): _pmx_iface.append({'name': iface['iface'], 'mtu' : int(iface.get('mtu', 1500)), 'tagged_vlans': _tagged_vlans}) if not len(ntb_iface): if len(_tagged_vlans): - ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='bridge', mtu=int(iface.get('mtu', 1500)), mode='tagged', tagged_vlans=[nb.ipam.vlans.get(vid=_tagged_vlans[0]).id]) + _nb_vlan = get_or_create_vlan(_tagged_vlans[0]) + ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='bridge', mtu=int(iface.get('mtu', 1500)), mode='tagged', tagged_vlans=[_nb_vlan.id]) else: ntb_iface = nb.dcim.interfaces.create(device=netbox_node.id, name=iface['iface'], type='bridge', mtu=int(iface.get('mtu', 1500))) else: @@ -209,7 +230,8 @@ def interfaces(proxmox, netbox_node, proxmox_json): ntb_if = next((_if for _if in _ntb_iface if _if['name'] == iface['iface']), None) if pmx_if != ntb_if: if len(pmx_if['tagged_vlans']): - nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu'], 'mode': 'tagged', 'tagged_vlans': [nb.ipam.vlans.get(vid=_tagged_vlans[0]).id]}]) + _nb_vlan = get_or_create_vlan(_tagged_vlans[0]) + nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu'], 'mode': 'tagged', 'tagged_vlans': [_nb_vlan.id]}]) else: nb.dcim.interfaces.update([{'id': ntb_iface.id, 'mtu': pmx_if['mtu']}]) updated = True From 482c570f9a41618e5518e6e514095e666d43dd7c Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sat, 8 Jul 2023 11:11:17 +0200 Subject: [PATCH 6/7] fix(Multi server): Proxmox password authentication Signed-off-by: devopstales <42894256+devopstales@users.noreply.github.com> --- netbox_proxbox/__init__.py | 2 +- netbox_proxbox/proxbox_api/plugins_config.py | 30 ++++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/netbox_proxbox/__init__.py b/netbox_proxbox/__init__.py index 5b409ba..bfb25f8 100755 --- a/netbox_proxbox/__init__.py +++ b/netbox_proxbox/__init__.py @@ -18,7 +18,7 @@ class ProxboxConfig(PluginConfig): 'user': 'root@pam', 'password': 'Strong@P4ssword', 'token': { - 'name': 'tokenID', + 'name': 'proxbox', 'value': '039az154-23b2-4be0-8d20-b66abc8c4686' }, 'ssl': False diff --git a/netbox_proxbox/proxbox_api/plugins_config.py b/netbox_proxbox/proxbox_api/plugins_config.py index df7e70d..881f421 100755 --- a/netbox_proxbox/proxbox_api/plugins_config.py +++ b/netbox_proxbox/proxbox_api/plugins_config.py @@ -109,19 +109,26 @@ def get_proxmox_session(PROXMOX_SETTING): PROXMOX_USER = PROXMOX_SETTING.get("user", DEFAULT_PROXMOX_USER) PROXMOX_PASSWORD = PROXMOX_SETTING.get("password", DEFAULT_PROXMOX_PASSWORD) - PROXMOX_TOKEN = PROXMOX_SETTING.get("token", DEFAULT_PROXMOX_TOKEN) - if PROXMOX_TOKEN != None: - PROXMOX_TOKEN_NAME = PROXMOX_TOKEN.get("name", DEFAULT_PROXMOX_TOKEN_NAME) - PROXMOX_TOKEN_VALUE = PROXMOX_TOKEN.get("value", DEFAULT_PROXMOX_TOKEN_VALUE) - output = { 'PROXMOX': PROXMOX, 'PROXMOX_PORT': PROXMOX_PORT, 'PROXMOX_SSL': PROXMOX_SSL, - 'PROXMOX_TOKEN': PROXMOX_TOKEN, - 'PROXMOX_TOKEN_NAME': PROXMOX_TOKEN_NAME, - 'PROXMOX_TOKEN_VALUE': PROXMOX_TOKEN_VALUE + 'PROXMOX_TOKEN': None, + 'PROXMOX_TOKEN_NAME': None, + 'PROXMOX_TOKEN_VALUE': None } + + PROXMOX_TOKEN = PROXMOX_SETTING.get("token", DEFAULT_PROXMOX_TOKEN) + if PROXMOX_PASSWORD is None and PROXMOX_TOKEN is not None: + PROXMOX_TOKEN_NAME = PROXMOX_TOKEN.get("name", DEFAULT_PROXMOX_TOKEN_NAME) + PROXMOX_TOKEN_VALUE = PROXMOX_TOKEN.get("value", DEFAULT_PROXMOX_TOKEN_VALUE) + output["PROXMOX_TOKEN"] = PROXMOX_TOKEN + output["PROXMOX_TOKEN_NAME"] = PROXMOX_TOKEN_NAME + output["PROXMOX_TOKEN_VALUE"] = PROXMOX_TOKEN_VALUE + else: + PROXMOX_TOKEN_NAME = None + PROXMOX_TOKEN_VALUE = None + #################################################################################################### # # # WITH PLUGIN CONFIGURED, STARTS BOTH PROXMOX AND NETBOX SESSION # @@ -132,7 +139,7 @@ def get_proxmox_session(PROXMOX_SETTING): # PROXMOX SESSION # # Check if token was provided - if PROXMOX_TOKEN_VALUE != None and len(PROXMOX_TOKEN_VALUE) > 0: + if PROXMOX_TOKEN_VALUE is not None and len(PROXMOX_TOKEN_VALUE) > 0: try: if PROXMOX_SSL == False: # DISABLE SSL WARNING @@ -156,6 +163,11 @@ def get_proxmox_session(PROXMOX_SETTING): # If token not provided, start session using user and passwd else: try: + if PROXMOX_SSL == False: + # DISABLE SSL WARNING + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + # Start PROXMOX session using USER CREDENTIALS PROXMOX_SESSION = ProxmoxAPI( PROXMOX, From 0002900d22149e0bacb10732e95c4b56195db4d2 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sun, 9 Jul 2023 10:49:28 +0200 Subject: [PATCH 7/7] feat(Multi server): Sync single cluster https://github.com/netdevopsbr/netbox-proxbox/issues/33 Signed-off-by: devopstales <42894256+devopstales@users.noreply.github.com> --- netbox_proxbox/proxbox_api/update.py | 29 +++++++++++++++++-- .../templates/netbox_proxbox/home.html | 19 +++++++++++- netbox_proxbox/urls.py | 4 ++- netbox_proxbox/views.py | 26 +++++++++++++++-- 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/netbox_proxbox/proxbox_api/update.py b/netbox_proxbox/proxbox_api/update.py index 5270fd9..80139c8 100755 --- a/netbox_proxbox/proxbox_api/update.py +++ b/netbox_proxbox/proxbox_api/update.py @@ -475,7 +475,7 @@ def run_process_in_thread(proxmox_session, key, result, index, **kwargs): def all(**kwargs): run_with_threads = kwargs.get("run_with_threads", True) start_time = time.time() - result = [] + results = [] if run_with_threads: print("Start process with threading") threads = [None] * len(proxmox_sessions) @@ -498,14 +498,37 @@ def all(**kwargs): session = proxmox_sessions[key] print("Processing data for: {0}".format(key)) output = process_all_in_session(session, **kwargs) - result.append(output) + results.append(output) except Exception as e: message = "OS error: {0}".format(e) print(message) output = { 'message': message } - result.append(output) + results.append(output) + + print("--- %s seconds ---" % (time.time() - start_time)) + return results + +def single(**kwargs): + proxmox_domain = kwargs.get("proxmox_domain", False) + start_time = time.time() + results = [] + for key in proxmox_sessions: + if key == proxmox_domain: + print("Start process Sequential") + try: + session = proxmox_sessions[key] + print("Processing data for: {0}".format(key)) + output = process_all_in_session(session, **kwargs) + results.append(output) + except Exception as e: + message = "OS error: {0}".format(e) + print(message) + output = { + 'message': message + } + results.append(output) print("--- %s seconds ---" % (time.time() - start_time)) return results diff --git a/netbox_proxbox/templates/netbox_proxbox/home.html b/netbox_proxbox/templates/netbox_proxbox/home.html index 1c64c40..fd4afb6 100644 --- a/netbox_proxbox/templates/netbox_proxbox/home.html +++ b/netbox_proxbox/templates/netbox_proxbox/home.html @@ -85,7 +85,24 @@

{% endif %} -
+
+ {% if px.domain %} +
+ + + Proxmox Update +
+ {% else %} + Proxmox Update + {% endif %}