Skip to content
This repository has been archived by the owner on Sep 12, 2024. It is now read-only.

Commit

Permalink
Added support for python 3 for DIY DSC (#789)
Browse files Browse the repository at this point in the history
Summary of changes included as a part of the PR:

Porting changes (Fixes) to run all scripts on python 3
Updated DSC default resources offered under nx to run on python 3.
Added additional logging to all python scripts.
Fixes for bugs that were introduced as a part of DROP OMI Depenedency.
Description of the changes:
As a part of porting the code we had decided to keep the code for python3 in a seperate forlder.(Most of these changes were done as a part of OMSConfig Py3 support PR). The updated folder for python scripts is as follows:

python2 : /opt/microsoft/dsc/Scripts
python3: /opt/microsoft/dsc/Scripts/python3
The python version of the machine will be determined by checking the existance of python2 and python3 command. In this version python2 takes precidence if both python 2 and 3 are present. (This is done to maintain compabality for older distros.)

Instead of just logging success and failure a lot of debug and detailed logs ave been added. Going forward DSC will have two log files:

/var/opt/omi/log/dsc.log
/var/opt/omi/log/dscdetailed.log
A lot of issues were discovered for the master branch, as the last release was done years back. I have tried to fix most of them. The issue were as a part of the DROP OMI DEPENDECY pr, which was only intended for omsconfig and ended up breaking DIY DSC.

Testing:
Manually tested for sanity and happy path secnarios on both python 2 and 3.
Distro testing is in progress using the test suite.
Bug bas planned for this week.


* Initial Changes to run locally

* Update InstallModule.py

* Updated helper scripts to account for new paths

* Updated providers to get invoked with python3 instead of python

* Fix FindFirstFile Failed bug that was introduced in 2019

* Fixed path to omicli for scripts

* Fixed the bug causing test to fail, it was introduced by a 2019 change DROP OMI DEPENDENCY

* Update CAEngine.c

* Added logs to python scripts

* Updated logging to capture python major version

* Added logs around compare in nxFile

* Removed logging from import module to get rid of circular dependency

* Removed logging from RegenerateInit to remove circular dependency

* Update RegenerateInitFiles.py

* Fixed comparison in case of python 3 for nxFile

* Updated imports for loggig to work

* Using absolute path instead of relative paths for DSCLogger

* Update Makefile to respect version provided in Build Repo

* Updated paths

* Updated Register.py

* Updated to remove warnings thrown for 'is'

* Update if such that we check instance details before reginstance

* Additioal logs to debug a failure in get configuration will revert the ones not required

* Removed additional logging (These were just for debugging and not required to be released)

* Removed additional logs

* Updating content to string as OMI is not able to handle bytes

* Resolving sekhar's comments
  • Loading branch information
Jinesh Shah authored May 10, 2021
1 parent 346c0bd commit f0e0a53
Show file tree
Hide file tree
Showing 36 changed files with 252 additions and 81 deletions.
20 changes: 7 additions & 13 deletions LCM/dsc/engine/ConsistencyInvoker/ConsistencyInvoker.c
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,15 @@ int main(int argc, char *argv[])

char* dscScriptPath = malloc(strlen(DSC_SCRIPT_PATH) + 1);
dscScriptPath = strcpy(dscScriptPath, DSC_SCRIPT_PATH);

//if oms config check to be added
if(strstr(dscScriptPath, "omsconfig")!= NULL)
{
pythonCommand = getPythonProvider();

if(strcmp(pythonCommand, PYTHON3_COMMAND) == 0)
{
dscScriptPath = realloc(dscScriptPath, strlen(dscScriptPath) + strlen("/python3") + 1 );
dscScriptPath = strcat(dscScriptPath, "/python3");
}
}

pythonCommand = getPythonProvider();

if(strcmp(pythonCommand, PYTHON3_COMMAND) == 0)
{
dscScriptPath = realloc(dscScriptPath, strlen(dscScriptPath) + strlen("/python3") + 1 );
dscScriptPath = strcat(dscScriptPath, "/python3");
}

int fullCommandLength = strlen(pythonCommand) + 1 + strlen(dscScriptPath) + 1 + strlen(PYTHON_SCRIPT_NAME) + 1;
char fullCommand[fullCommandLength];

Expand All @@ -59,7 +54,6 @@ int main(int argc, char *argv[])
}


// I may need to move this method in some file which is accessible to all other files in the project.
char* getPythonProvider()
{
int buffer_length = 128;
Expand Down
2 changes: 1 addition & 1 deletion LCM/dsc/engine/ModuleLoader/ModuleLibrary/ModuleHandler.c
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,6 @@ MI_Result GetModuleLoader( _In_ MI_Application *miApp,
#if defined(BUILD_OMS)
if (g_DscHost == MI_FALSE)
{
#endif
// Get the Registration information from shared objects
r = GetRegistrationInstanceFromSharedObjects(NULL, miApp, de, options, strictOptions, &miClassArray, &miInstanceArray, extendedError);
if( r != MI_RESULT_OK)
Expand All @@ -482,6 +481,7 @@ MI_Result GetModuleLoader( _In_ MI_Application *miApp,
CleanUpInstanceCache(&miInstanceArray);
return r;
}
#endif
// No need to do this after removing OMI since we will discover resources directly from shared objects (except for DIY DSC)
/*Perform registration against schema validation*/
Expand Down
13 changes: 9 additions & 4 deletions LCM/dsc/engine/ca/CAInfrastructure/CAEngine.c
Original file line number Diff line number Diff line change
Expand Up @@ -1286,10 +1286,11 @@ MI_Result MoveToDesiredState(_In_ ProviderCallbackContext *provContext,
if (
#if defined(BUILD_OMS)
g_DscHost == MI_FALSE ||
Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
#else
Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
Tcscasecmp(regInstance->classDecl->name, BASE_REGISTRATION_WMIV2PROVIDER) == 0 ||
#endif
Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
Tcscasecmp(MSFT_LOGRESOURCENAME, instance->classDecl->name) == 0
)
{
Expand Down Expand Up @@ -1910,15 +1911,16 @@ MI_Result GetGetMethodResult(_In_ MI_Operation *operation,
*extendedError = NULL; // Explicitly set *extendedError to NULL as _Outptr_ requires setting this at least once.

*outputInstance = NULL;

/*Get the operation result*/
r = MI_Operation_GetInstance(operation, &outInstance, &moreResults, &result, &errorMessage, &completionDetails);
if( result != MI_RESULT_OK)
{
DSC_LOG_INFO("MI_Operation_GetInstance inside GetGetMethodResult failed result not ok");
r = result;
}
if( r != MI_RESULT_OK)
{
DSC_LOG_INFO("MI_Operation_GetInstance inside GetGetMethodResult failed r not ok");
if( completionDetails != NULL)
{
innerR = DSC_MI_Instance_Clone( completionDetails, extendedError);
Expand All @@ -1927,6 +1929,7 @@ MI_Result GetGetMethodResult(_In_ MI_Operation *operation,
{
r = GetCimMIError(r, extendedError,ID_CAINFRA_GETINSTANCE_FAILED);
}

return r;
}

Expand Down Expand Up @@ -2021,18 +2024,21 @@ MI_Result Get_WMIv2Provider(_In_ ProviderCallbackContext *provContext,
return GetCimMIError(r, extendedError,ID_CAINFRA_GET_NEWAPPLICATIONINSTANCE_FAILED);
}
value.instance = instance;

r = DSC_MI_Instance_AddElement(params, OMI_BaseResource_Method_InputResource, &value, MI_INSTANCE, 0 );
if( r != MI_RESULT_OK)
{
MI_Instance_Delete(params);
return GetCimMIError(r, extendedError,ID_CAINFRA_GET_ADDELEM_FAILED);
}

r = MI_Application_NewOperationOptions(miApp, MI_FALSE, &sessionOptions);
if( r != MI_RESULT_OK )
{
return GetCimMIError(r, extendedError,ID_CAINFRA_GET_NEWOPERATIONOPTIONS_FAILED);
}
valueOperationOptions.string=g_ConfigurationDetails.jobGuidString;

r =MI_OperationOptions_SetCustomOption(&sessionOptions,DSC_JOBIDSTRING,MI_STRING,&valueOperationOptions,MI_FALSE);
if( r != MI_RESULT_OK)
{
Expand All @@ -2041,17 +2047,16 @@ MI_Result Get_WMIv2Provider(_In_ ProviderCallbackContext *provContext,
}

/* Perform Get*/

MI_Session_Invoke(miSession, 0, &sessionOptions, provNamespace,
instance->classDecl->name, OMI_BaseResource_GetMethodName,
NULL, params, &callbacks,&operation);

r = GetGetMethodResult(&operation, outputInstance , extendedError);
MI_Instance_Delete(params);
MI_OperationOptions_Delete(&sessionOptions);
MI_Operation_Close(&operation);
if( r != MI_RESULT_OK)
{
DSC_LOG_INFO("GetGetMethodResult r is not ok");
return r;
}
}
Expand Down
18 changes: 13 additions & 5 deletions LCM/scripts/GetDscConfiguration.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,31 @@
import os
import os.path
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
from imp import load_source
from os.path import dirname, isfile, join, realpath
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
from imp import load_source
from os.path import dirname, isfile, join, realpath
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
from sys import argv


pathToCurrentScript = realpath(__file__)
pathToCommonScriptsFolder = dirname(pathToCurrentScript)

DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
LG = nxDSCLog.DSCLog

helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
helperlib = load_source('helperlib', helperLibPath)

omi_bindir = "<CONFIG_BINDIR>"
omicli_path = omi_bindir + "/omicli"
omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
dsc_host_output_path = join(dsc_host_base_path, 'output')
dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')

LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))

if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))

Expand Down Expand Up @@ -93,4 +100,5 @@
print(stdout)
print(stderr)

LG().Log("DEBUG", "End of script logic for " + argv[0] + " runing with python " + str(sys.version_info.major))

17 changes: 12 additions & 5 deletions LCM/scripts/GetDscLocalConfigurationManager.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,30 @@
import os
import os.path
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
from imp import load_source
from os.path import dirname, isfile, join, realpath
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
from imp import load_source
from os.path import dirname, isfile, join, realpath
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
from sys import argv

pathToCurrentScript = realpath(__file__)
pathToCommonScriptsFolder = dirname(pathToCurrentScript)

DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
LG = nxDSCLog.DSCLog

helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
helperlib = load_source('helperlib', helperLibPath)

omi_bindir = "<CONFIG_BINDIR>"
omicli_path = omi_bindir + "/omicli"
omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
dsc_host_output_path = join(dsc_host_base_path, 'output')
dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')

LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))

if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))

Expand Down Expand Up @@ -93,4 +99,5 @@
print(stdout)
print(stderr)

LG().Log("DEBUG", "End of script logic for " + argv[0] + " runing with python " + str(sys.version_info.major))

4 changes: 2 additions & 2 deletions LCM/scripts/InstallModule.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import subprocess
import sys
import platform
from os.path import basename, dirname, join, realpath, split
from os.path import basename, dirname, join, realpath, split

pathToCurrentScript = realpath(__file__)
pathToCommonScriptsFolder = dirname(pathToCurrentScript)
Expand Down Expand Up @@ -357,4 +357,4 @@ def main(args):
regenerateDscPythonScriptInitFiles()

if __name__ == "__main__":
main(sys.argv[1:])
main(sys.argv[1:])
9 changes: 8 additions & 1 deletion LCM/scripts/PerformRequiredConfigurationChecks.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
#!/usr/bin/python
import sys
from imp import load_source
from os.path import dirname, join, realpath, isfile
from subprocess import PIPE, Popen
from sys import exc_info, exit, version_info
from sys import exc_info, exit, version_info, argv
from traceback import format_exc
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
Expand All @@ -11,6 +12,10 @@
pathToCurrentScript = realpath(__file__)
pathToCommonScriptsFolder = dirname(pathToCurrentScript)

DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
LG = nxDSCLog.DSCLog

helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
helperlib = load_source('helperlib', helperLibPath)

Expand Down Expand Up @@ -111,4 +116,6 @@ def run_perform_required_configuration_checks():
print(stdout)

if __name__ == "__main__":
LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
main()
LG().Log("DEBUG", "End of Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
4 changes: 2 additions & 2 deletions LCM/scripts/RegenerateInitFiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
omi_libdir + "/2.6x-2.7x/Scripts",
omi_libdir + "/3.x/Scripts"]


for current_dir in script_dirs:
out_init = "__all__="
py_files = glob.glob(current_dir + "/*.py")
Expand All @@ -24,5 +25,4 @@
py_files_basename.append(current_basename[:-3])

out_init = out_init + str(py_files_basename)
open(current_dir + "/__init__.py", "w").write(out_init)

open(current_dir + "/__init__.py", "w").write(out_init)
15 changes: 15 additions & 0 deletions LCM/scripts/Register.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
import os.path
import tempfile
import shutil
from sys import argv
from imp import load_source
from os.path import dirname, join, realpath

def usage():
print("""Usage: Register.py [OPTIONS]
Expand All @@ -18,6 +21,16 @@ def usage():
--Help
""")


pathToCurrentScript = realpath(__file__)
pathToCommonScriptsFolder = dirname(pathToCurrentScript)

DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
LG = nxDSCLog.DSCLog

LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))

# Apply a DSC meta configuration based on a template
Variables = dict()

Expand Down Expand Up @@ -217,3 +230,5 @@ def usage():
os.system("<DSC_SCRIPT_PATH>/SetDscLocalConfigurationManager.py -configurationmof " + meta_path)

shutil.rmtree(tempdir)

LG().Log("DEBUG", "End of script logic for " + argv[0] + " runing with python " + str(sys.version_info.major))
9 changes: 9 additions & 0 deletions LCM/scripts/RemoveModule.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,16 @@
import platform
import imp
from os.path import dirname, join, realpath
from sys import argv
from imp import load_source

pathToCurrentScript = realpath(__file__)
pathToCommonScriptsFolder = dirname(pathToCurrentScript)

DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
LG = nxDSCLog.DSCLog

helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
helperlib = imp.load_source('helperlib', helperLibPath)
fullPathDSCLogger = os.path.join(pathToCommonScriptsFolder, 'nxDSCLog.py')
Expand Down Expand Up @@ -196,4 +203,6 @@ def main(args):
shutil.rmtree(modulePath)

if __name__ == "__main__":
LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
main(sys.argv[1:])
LG().Log("DEBUG", "End of Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
13 changes: 11 additions & 2 deletions LCM/scripts/RestoreConfiguration.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,28 @@
from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
from time import sleep
from sys import argv


pathToCurrentScript = realpath(__file__)
pathToCommonScriptsFolder = dirname(pathToCurrentScript)

DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
LG = nxDSCLog.DSCLog

helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
helperlib = load_source('helperlib', helperLibPath)

omi_bindir = "<CONFIG_BINDIR>"
omicli_path = omi_bindir + "/omicli"
omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
dsc_host_output_path = join(dsc_host_base_path, 'output')
dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')

LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))

if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))

Expand Down Expand Up @@ -86,3 +93,5 @@

print(stdout)
print(stderr)

LG().Log("DEBUG", "End of script logic for " + argv[0] + " runing with python " + str(sys.version_info.major))
8 changes: 8 additions & 0 deletions LCM/scripts/SetDscLocalConfigurationManager.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,16 @@
from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
from time import sleep
import signal
import sys


pathToCurrentScript = realpath(__file__)
pathToCommonScriptsFolder = dirname(pathToCurrentScript)

DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
nxDSCLog = load_source('nxDSCLog', DSCLogPath)
LG = nxDSCLog.DSCLog

helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
helperlib = load_source('helperlib', helperLibPath)

Expand Down Expand Up @@ -170,4 +176,6 @@ def signal_handler(signalNumber, frame):
if __name__ == "__main__":
# register the SIGTERM handler
signal.signal(signal.SIGTERM, signal_handler)
LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
main(argv)
LG().Log("DEBUG", "End of Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
Loading

0 comments on commit f0e0a53

Please sign in to comment.