diff --git a/Convert_YANG_To_XML/.meta_Convert_YANG_To_XML.xml b/Convert_YANG_To_MicroService/.meta_Convert_YANG_To_MicroService.xml old mode 100755 new mode 100644 similarity index 60% rename from Convert_YANG_To_XML/.meta_Convert_YANG_To_XML.xml rename to Convert_YANG_To_MicroService/.meta_Convert_YANG_To_MicroService.xml index 384e424ca..6dcf9c2d6 --- a/Convert_YANG_To_XML/.meta_Convert_YANG_To_XML.xml +++ b/Convert_YANG_To_MicroService/.meta_Convert_YANG_To_MicroService.xml @@ -3,11 +3,11 @@ DISPLAYNAME - Convert_YANG_To_XML.xml + Convert_YANG_To_MicroService.xml DATE_MODIFICATION - 1617358606722 + 1619793743117 REPOSITORY @@ -15,23 +15,11 @@ DATE_CREATION - 1617358606714 - - - MODEL - 0 + 1619793743117 TAG - - FILE_TYPE - text - - - MANUFACTURER - 0 - TYPE FILE diff --git a/Convert_YANG_To_XML/.meta_Process_convert_yang_to_xml b/Convert_YANG_To_MicroService/.meta_Process_Delete old mode 100755 new mode 100644 similarity index 82% rename from Convert_YANG_To_XML/.meta_Process_convert_yang_to_xml rename to Convert_YANG_To_MicroService/.meta_Process_Delete index 4fc32e2ce..1912c73a8 --- a/Convert_YANG_To_XML/.meta_Process_convert_yang_to_xml +++ b/Convert_YANG_To_MicroService/.meta_Process_Delete @@ -3,11 +3,11 @@ DISPLAYNAME - Process_convert_yang_to_xml + Process_Delete DATE_MODIFICATION - 1616080840816 + 1617349254149 REPOSITORY @@ -15,7 +15,7 @@ DATE_CREATION - 1616080840812 + 1617349254142 TAG diff --git a/Convert_YANG_To_MicroService/.meta_Process_convert_yang_to_microservice b/Convert_YANG_To_MicroService/.meta_Process_convert_yang_to_microservice new file mode 100644 index 000000000..ed97b600f --- /dev/null +++ b/Convert_YANG_To_MicroService/.meta_Process_convert_yang_to_microservice @@ -0,0 +1,31 @@ + + + + + DISPLAYNAME + Process_Convert_XML_to_MicroService + + + DATE_MODIFICATION + 1617785102777 + + + REPOSITORY + Process + + + DATE_CREATION + 1617785102709 + + + TAG + + + TYPE + DIRECTORY + + + COMMENT + + + diff --git a/Convert_YANG_To_XML/.meta_Process_create_instance b/Convert_YANG_To_MicroService/.meta_Process_create_instance old mode 100755 new mode 100644 similarity index 100% rename from Convert_YANG_To_XML/.meta_Process_create_instance rename to Convert_YANG_To_MicroService/.meta_Process_create_instance diff --git a/Convert_YANG_To_XML/Convert_YANG_To_XML.xml b/Convert_YANG_To_MicroService/Convert_YANG_To_MicroService.xml similarity index 54% rename from Convert_YANG_To_XML/Convert_YANG_To_XML.xml rename to Convert_YANG_To_MicroService/Convert_YANG_To_MicroService.xml index 4fcf381a1..64c515ed2 100644 --- a/Convert_YANG_To_XML/Convert_YANG_To_XML.xml +++ b/Convert_YANG_To_MicroService/Convert_YANG_To_MicroService.xml @@ -14,31 +14,41 @@ + - + + - + Create Converter Instance CREATE 5 - /opt/fmc_repository/Process/Convert_YANG_To_XML/Process_create_instance/Tasks + /opt/fmc_repository/Process/Convert_YANG_To_MicroService/Process_create_instance/Tasks create_instance - - Convert Yang files to XML + + Convert Yang files to MicroService UPDATE 5 - /opt/fmc_repository/Process/Convert_YANG_To_XML/Process_convert_yang_to_xml/Tasks + /opt/fmc_repository/Process/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks Convert Yang files to XML + + /opt/fmc_repository/Process/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks + Update XML File for MS + + + /opt/fmc_repository/Process/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks + Convert XML to MicroService + - + Delete DELETE 5 @@ -47,11 +57,11 @@ Convert Yang files into xml files - yang_list.0.yang + service_id python 10000 5 - YANG to XML Converter + YANG to MicroService Converter tool|yang diff --git a/Convert_YANG_To_XML/Process_Delete/.meta_Tasks b/Convert_YANG_To_MicroService/Process_Delete/.meta_Tasks old mode 100755 new mode 100644 similarity index 100% rename from Convert_YANG_To_XML/Process_Delete/.meta_Tasks rename to Convert_YANG_To_MicroService/Process_Delete/.meta_Tasks diff --git a/Convert_YANG_To_XML/Process_convert_yang_to_xml/.meta_Tasks b/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/.meta_Tasks old mode 100755 new mode 100644 similarity index 100% rename from Convert_YANG_To_XML/Process_convert_yang_to_xml/.meta_Tasks rename to Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/.meta_Tasks diff --git a/Convert_YANG_To_XML/Process_convert_yang_to_xml/Tasks/.meta_Task_transform_yang_files_to_xml.py b/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/.meta_Task_transform_yang_files_to_xml.py old mode 100755 new mode 100644 similarity index 100% rename from Convert_YANG_To_XML/Process_convert_yang_to_xml/Tasks/.meta_Task_transform_yang_files_to_xml.py rename to Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/.meta_Task_transform_yang_files_to_xml.py diff --git a/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_Convert_xml_to_MicroService.py b/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_Convert_xml_to_MicroService.py new file mode 100644 index 000000000..931e17875 --- /dev/null +++ b/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_Convert_xml_to_MicroService.py @@ -0,0 +1,195 @@ +import copy +import json +import re +import os +import subprocess + + +from xml.etree.ElementTree import ElementTree +from xml.etree.ElementTree import Element +from xml.etree.ElementTree import SubElement +import xml.etree.ElementTree as ET +import numpy as np +#from re import search +from msa_sdk import constants +from msa_sdk.variables import Variables +from msa_sdk.msa_api import MSA_API +import requests +from xml.dom import minidom +#from xml.dom.ext.reader.Sax2 import FromXmlStream +from msa_sdk.device import Device + +dev_var = Variables() +context = Variables.task_call(dev_var) +yang_fields_without_child = context['yang_fields_without_child_hash'].keys() + +msa_object = MSA_API() +msa_object.action = 'Create MS object' +msa_object.path = "/repository/v2/resource/microservice" + + +# Build the curl command to create the Micro Service + +ms_name = str(os.path.basename(context['main_yang_file']).replace(context['yangs_extension'],'')) +ms_filename = "CommandDefinition/microservices/" + ms_name + '.xml' + + +curl_ms_creation_xml = """ { + "information": { + "displayName": """+ '"'+ ms_name + """", + "icon": "none", + "description": """ +'"'+ ms_name + """", + "category": "Default", + "displayField": "object_id", + "order": 0, + "visibility": "5", + "configType": "netconf" + }, + "variables": { + "variable": [ + { + "displayName": "object_id", + "fullDisplayName": "", + "displayNameHeader": "", + "displayOrder": 0, + "name": "params.object_id", + "description": "", + "type": "String", + "visible": true, + "userLocked": false, + "onlyDetailView": false, + "grouped": false, + "groupDisplayName": "", + "groupSeparator": "", + "mandatory": false, + "editable": false, + "arrayCanAdd": true, + "arrayCanRemove": true, + "arrayCanMove": true, + "arrayCanEdit": true + } """ + +for yang_good_field in yang_fields_without_child: + curl_ms_creation_xml = curl_ms_creation_xml + """, + { + "displayName": """ + '"'+ yang_good_field + """", + "fullDisplayName": """ + '"'+ yang_good_field + """", + "displayNameHeader": "", + "displayOrder": 0, + "name": "params.""" + yang_good_field + """", + "description": "", + "type": "String", + "visible": true, + "userLocked": false, + "onlyDetailView": false, + "grouped": false, + "groupDisplayName": "", + "groupSeparator": "", + "mandatory": false, + "editable": false, + "arrayCanAdd": true, + "arrayCanRemove": true, + "arrayCanMove": true, + "arrayCanEdit": true + } """ + +curl_ms_creation_xml = curl_ms_creation_xml + """ + ], + "frozen": 0 + }, + "command": [ + { + "operation": null, + "name": "IMPORT", + "postTemplate": null, + "parser": { + "section": [ + { + "xpath": "" + } + ], + "lines": {} + } + }, + { + "operation": " """ +yang_create_operations = '' +#get Create part from xml +f = open(context['xml_create_output_file'], "r") +string_create_part = f.read() +string_create_part = string_create_part.replace('\"','\\"') #protect " in the string +string_create_part = string_create_part.replace('\n','\\n') #protect end of line for json convertion +yang_create_operations = yang_create_operations + string_create_part + '" ' + +curl_ms_creation_xml = curl_ms_creation_xml + yang_create_operations + """, + "name": "CREATE" + }, + { + "operation": null, + "name": "READ" + }, + { + "operation": " """ +curl_ms_creation_xml = curl_ms_creation_xml + yang_create_operations + """, + "name": "IMPORT" + }, + { + "operation": null, + "name": "UPDATE" + }, + { + "operation": null, + "name": "CONSTRAINT" + }, + { + "operation": null, + "name": "DELETE" + }, + { + "operation": null, + "name": "LIST" + } + ], + "example": { + "content": " """ +curl_ms_creation_xml = curl_ms_creation_xml + yang_create_operations + """ + }, + "metaInformationList": [ + { + "type": "FILE", + "uri": """+ '"'+ ms_filename + """", + "file": true, + "name": """+ '"'+ ms_name + '.xml' + """", + "displayName": """+ '"'+ ms_name + '.xml' + """", + "repositoryName": "CommandDefinition", + "parentURI": "CommandDefinition", + "fileType": "text", + "tag": "string", + "comment": "string", + "modelId": 137, + "vendorId": 1 + } + ] +} +""" + + +context.update(curl_ms_creation_xml=curl_ms_creation_xml) + +#Uncomment following lines to degub, check the contain of following file, it should be convertible into one json array (it it is the curl request parameter) +#fw = open(context['xml_create_output_file']+'_curl_content.json', "w") +#fw.write(curl_ms_creation_xml) +#fw.close() + + +parameters_json = json.loads(curl_ms_creation_xml) +#Create the MicroService with Culr requests +msa_object.call_post(parameters_json) + + +context['micro_service_file'] = ms_filename + +ret = MSA_API.process_content(constants.ENDED, ' New MicroService "'+ ms_name + '" created', context, True) +print(ret) + + diff --git a/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_Update_XML.py b/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_Update_XML.py new file mode 100644 index 000000000..ef43f80cf --- /dev/null +++ b/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_Update_XML.py @@ -0,0 +1,100 @@ +import copy +import json +import re +import os +import subprocess + + +from xml.etree.ElementTree import ElementTree +from xml.etree.ElementTree import Element +from xml.etree.ElementTree import SubElement +import xml.etree.ElementTree as ET +import numpy as np +#from re import search +from msa_sdk import constants +from msa_sdk.variables import Variables +from msa_sdk.msa_api import MSA_API +from xml.dom import minidom +#from xml.dom.ext.reader.Sax2 import FromXmlStream + +dev_var = Variables() +context = Variables.task_call(dev_var) +xml_output_file = context['xml_output_file'] + +''' #To used if you want filter the variable which are in YANG file with ' leaf ' +yang_good_fields_hash = {} +yang_good_fields = [] +pyang_command = ' grep " leaf " ' + str(context['main_yang_file']) +try: + output = subprocess.check_output(pyang_command, shell=True, stderr=subprocess.STDOUT) +except subprocess.CalledProcessError: + ret = MSA_API.process_content(constants.FAILED, 'Error:' + stderr, context, True) + print(ret) +yang_lines=[] +for line in output.splitlines( ): + yang_lines.append(line) + field = re.search(' leaf (\S+) ', str(line), re.IGNORECASE) + if field: + yang_good_fields_hash[field.group(1)] = 1 #used hash array to get only one value per fieldname +yang_good_fields = yang_good_fields_hash.keys() +context['yang_good_fields_hash'] = yang_good_fields_hash +''' + +yang_fields_without_child_hash = {} + +from lxml import etree, objectify + +metadata = xml_output_file +parser = etree.XMLParser(attribute_defaults=True, huge_tree=True, ns_clean=False, remove_blank_text=False) +newtree = etree.parse(metadata, parser) +newroot = newtree.getroot() +#objectify.deannotate(newroot, cleanup_namespaces=True) # Remove namespace xmlns=xxxx +if (newroot.tag.find('}config')): + #remove first parent element 'config' + newtree = etree.ElementTree(newroot[0]) + +res = '' +for elem in newroot.getiterator(): + if not hasattr(elem.tag, 'find'): continue # (1) + i = elem.tag.find('}') + if i >= 0: + elem.tag = elem.tag[i+1:] + +i=0 +parent_yangfield_nochild = {} # list of parent field who has some chidren field which are in the yang file and has no child. + +for element in newroot.iter(): + children = list(element) + if (i>0): + parent = element.getparent() + if len(children): + res = res + ";\n tag="+ str(element.tag) + ", len="+str(len(children))+ ", parent=" + str(parent.tag) + else: + res = res + ";\n tag="+ str(element.tag) + ", NO_child" + ", parent=" + str(parent.tag) + #find all parents + all_parents=[] + grandparent = element.getparent() + #grandparent = grandparent.getparent() # remove first bad parent + while grandparent is not None: + all_parents.insert(0,grandparent.tag) #insert first element + grandparent = grandparent.getparent() + all_parents.pop(0) #remove the fisrt config element + new_tagname = '_'.join(all_parents) + '_' + element.tag + #element.tag = new_tagname + element.text = '{$params.' + new_tagname + '}' + yang_fields_without_child_hash[new_tagname] = 1 #used hash array to get only one value per fieldname + + i = i+1 + +xml_create_output_file = xml_output_file.replace(context['yangs_extension'],'')+'_created.xml' +context['xml_create_output_file'] = xml_create_output_file + +context['yang_fields_without_child_hash'] = yang_fields_without_child_hash + +newtree.write(xml_create_output_file, pretty_print=True, xml_declaration=False, encoding='UTF-8') + +ret = MSA_API.process_content(constants.ENDED, 'Created Part xml = '+xml_create_output_file , context, True) +print(ret) + + + diff --git a/Convert_YANG_To_XML/Process_convert_yang_to_xml/Tasks/Task_transform_yang_files_to_xml.py b/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_transform_yang_files_to_xml.py old mode 100755 new mode 100644 similarity index 77% rename from Convert_YANG_To_XML/Process_convert_yang_to_xml/Tasks/Task_transform_yang_files_to_xml.py rename to Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_transform_yang_files_to_xml.py index a1796fcb4..08998c173 --- a/Convert_YANG_To_XML/Process_convert_yang_to_xml/Tasks/Task_transform_yang_files_to_xml.py +++ b/Convert_YANG_To_MicroService/Process_convert_yang_to_microservice/Tasks/Task_transform_yang_files_to_xml.py @@ -13,6 +13,7 @@ dev_var = Variables() dev_var.add('yang_list.0.yang', var_type='String') dev_var.add('yang_list.0.is_selected', var_type='Boolean') +dev_var.add('yang_list.0.is_yangmainfile', var_type='Boolean') context = Variables.task_call(dev_var) @@ -47,27 +48,37 @@ # Retrieve selected yang_filename from the list. yang_filename = '' yang_filenames = [] +main_yang_file = '' for st in yang_list: - # "yang_list": [ { "yang": "/opt/fmc_repository/Datafiles/YANG/example-1.yang", "is_selected": "false" }, - # { "yang": "/opt/fmc_repository/Datafiles/YANG/example-2.yang", "is_selected": "false" } ] + # "yang_list": [ { "yang": "/opt/fmc_repository/Datafiles/YANG/example-1.yang", "is_selected": "false", "is_yangmainfile": "false" }, + # { "yang": "/opt/fmc_repository/Datafiles/YANG/example-2.yang", "is_selected": "false", "is_yangmainfile": "false" } ] + if st.get('is_selected') == True: if st.get('yang'): yang_filename = context.get('yangs_directory') + '/' + st.get('yang') yang_filenames.append(yang_filename) - + if st.get('is_yangmainfile') == True: + main_yang_file = yang_filename + else: ret = MSA_API.process_content(constants.FAILED, 'Selected yang filename is empty from the service instance context.', context, True) print(ret) context['yang_filenames'] = yang_filenames +if main_yang_file =='': + #take the first yang selected + context['main_yang_file'] = yang_filenames[0] + main_yang_file = yang_filenames[0] +else: + context['main_yang_file'] = main_yang_file if selected_number ==0: ret = MSA_API.process_content(constants.FAILED, 'No yang file selected', context, True) -xml_output_file = yang_filename.replace(context['yangs_extension'],'') + '_output.xml' +xml_output_file = main_yang_file.replace(context['yangs_extension'],'') + '_output.xml' context['xml_output_file'] = xml_output_file -yang_path = os.path.dirname(yang_filename) # run pyang in the given directorie to be able to load other yang generic library dependency +yang_path = os.path.dirname(main_yang_file) # run pyang in the given directorie to be able to load other yang generic library dependency pyang_command = ' cd "'+yang_path+'"; pyang -f sample-xml-skeleton --sample-xml-skeleton-doctype=config -o ' + xml_output_file + " " + " ".join(map(str, yang_filenames)) @@ -81,7 +92,6 @@ if len(yang_filenames) >1: - #ret = MSA_API.process_content(constants.ENDED, 'Yangs files "' + " ,".join(map(str, yang_filenames)) + '" are parsed successfully into "'+ xml_output_file + '"' , context, True) ret = MSA_API.process_content(constants.ENDED, 'New XML output file: "'+ xml_output_file + '"' , context, True) else: ret = MSA_API.process_content(constants.ENDED, 'New XML output file: "'+ xml_output_file + '"' , context, True) diff --git a/Convert_YANG_To_XML/Process_create_instance/.meta_Tasks b/Convert_YANG_To_MicroService/Process_create_instance/.meta_Tasks old mode 100755 new mode 100644 similarity index 100% rename from Convert_YANG_To_XML/Process_create_instance/.meta_Tasks rename to Convert_YANG_To_MicroService/Process_create_instance/.meta_Tasks diff --git a/Convert_YANG_To_XML/Process_create_instance/Tasks/.meta_Task_create_instance.py b/Convert_YANG_To_MicroService/Process_create_instance/Tasks/.meta_Task_create_instance.py old mode 100755 new mode 100644 similarity index 100% rename from Convert_YANG_To_XML/Process_create_instance/Tasks/.meta_Task_create_instance.py rename to Convert_YANG_To_MicroService/Process_create_instance/Tasks/.meta_Task_create_instance.py diff --git a/Convert_YANG_To_XML/Process_create_instance/Tasks/Task_create_instance.py b/Convert_YANG_To_MicroService/Process_create_instance/Tasks/Task_create_instance.py old mode 100755 new mode 100644 similarity index 91% rename from Convert_YANG_To_XML/Process_create_instance/Tasks/Task_create_instance.py rename to Convert_YANG_To_MicroService/Process_create_instance/Tasks/Task_create_instance.py index d6672994e..f6df9d429 --- a/Convert_YANG_To_XML/Process_create_instance/Tasks/Task_create_instance.py +++ b/Convert_YANG_To_MicroService/Process_create_instance/Tasks/Task_create_instance.py @@ -12,6 +12,7 @@ dev_var.add('yangs_extension', var_type='String') dev_var.add('yang_list.0.yang', var_type='String') dev_var.add('yang_list.0.is_selected', var_type='Boolean') +dev_var.add('yang_list.0.is_yangmainfile', var_type='Boolean') context = Variables.task_call(dev_var) @@ -53,13 +54,13 @@ # check if at least one yang file is available. If not exit and return failed. if not yang_list: - ret = MSA_API.process_content(constants.FAILED, 'No Yang files found from \'' + yangs_directory + '\' directory. with extension \'' + yangs_extension +'\'', context, True) + ret = MSA_API.process_content(constants.FAILED, 'No YANG files found from \'' + yangs_directory + '\' directory. with extension \'' + yangs_extension +'\'', context, True) print(ret) yang_list_restructured = [dict(yang=st, is_selected='false') for st in yang_list] #store in the context 'yang_list' context['yang_list'] = yang_list_restructured -ret = MSA_API.process_content(constants.ENDED, 'yang list acquisition is done successfully.', context, True) +ret = MSA_API.process_content(constants.ENDED, 'YANG list acquisition is done successfully.', context, True) print(ret) diff --git a/Convert_YANG_To_XML/README.md b/Convert_YANG_To_MicroService/README.md similarity index 65% rename from Convert_YANG_To_XML/README.md rename to Convert_YANG_To_MicroService/README.md index 329c93468..0c6cc33b5 100644 --- a/Convert_YANG_To_XML/README.md +++ b/Convert_YANG_To_MicroService/README.md @@ -1,11 +1,11 @@ -This WF is used to convert some YANG files into XML files. -It can convert many selected YANG files together into one xml file +This WF is used to convert some YANG files into one Micro Service +It can convert many selected YANG files together into one xml file and after create one Micro Service -If your YANG files need some the generic library dependency, you should put all generic library dependency files into the same directorie ans your YANG files. +If your YANG files need some the generic library dependency, you should put all generic library dependency files into the same directorie as your YANG files. The YANG file should be by default in the directory '/opt/fmc_repository/Datafiles/YANG' -The output xml will be created in the same directory. -Click at first on 'Create Converter Instance' to create on WF instance. And after click on 'Convert Yang files to XML' to convert files into XML. +The temporary output xml will be created in the same directory. +Click at first on 'Create Converter Instance' to create on WF instance. And after click on 'Convert Yang files to MicroService' to convert YANG files into Micro Service. Login to a live MSA as root and perform the following: @@ -14,10 +14,10 @@ Login to a live MSA as root and perform the following: git clone https://github.com/openmsa/Workflows OpenMSA_WF chown -R ncuser. OpenMSA_WF/ cd /opt/fmc_repository/Process - ln -sfn ../OpenMSA_WF/Convert_YANG_To_XML Convert_YANG_To_XML + ln -sfn ../OpenMSA_WF/Convert_YANG_To_MicroService Convert_YANG_To_MicroService Browse to the MSA GUI, open "Manage Repository". The new entry "OpenMSA" should be available and browsable under `Automation > Workflows`. -The WF path should be /opt/fmc_repository/Process/Convert_YANG_To_XML/ +The WF path should be /opt/fmc_repository/Process/Convert_YANG_To_MicroService/