diff --git a/Jenkinsfile b/Jenkinsfile index 7e381c6a..0d183a92 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ properties([parameters([string(name: 'projectVersion', defaultValue: 'v8.1.0a15' string(name: 'JENKINS_CE', defaultValue: 'jenkins.cern.ch', description: 'The CE definition to use (of DIRAC.Jenkins.ch, see CS for others)'), string(name: 'modules', defaultValue: '', description: 'to override what is installed, e.g. with https://github.com/$DIRAC_test_repo/DIRAC.git:::DIRAC:::$DIRAC_test_branch'), string(name: 'pip_install_options', defaultValue: '', description: 'options to pip install (e.g. --index-url=https://lhcb-repository.web.cern.ch/repository/pypi/simple)'), - string(name: 'pilot_options', defaultValue: '', description: 'any pilot option, e.g. --pythonVersion=2'), + string(name: 'pilot_options', defaultValue: '', description: 'any pilot option'), string(name: 'CSURL', defaultValue: 'https://lbcertifdirac70.cern.ch:9135/Configuration/Server', description: 'URL for CS'), string(name: 'DIRACSETUP', defaultValue: 'DIRAC-Certification', description: 'DIRAC setup'), string(name: 'PILOTJSON', defaultValue: 'pilot_oldSchema.json', description: 'other option: pilot_newSchema.json'), diff --git a/Pilot/dirac-install.py b/Pilot/dirac-install.py deleted file mode 100755 index df773133..00000000 --- a/Pilot/dirac-install.py +++ /dev/null @@ -1,1843 +0,0 @@ -#!/usr/bin/env python -""" -dirac-install.py for Pilot (simplified) -""" - -# pylint: skip-file - -from __future__ import unicode_literals, absolute_import, division, print_function - -import sys -import os -import getopt -import signal -import time -import stat -import shutil -import ssl -import hashlib - -try: - # For Python 3.0 and later - from urllib.request import urlopen, HTTPError, URLError -except ImportError: - # Fall back to Python 2's urllib2 - from urllib2 import urlopen, HTTPError, URLError -try: - str_type = basestring -except NameError: - str_type = str - -__RCSID__ = "$Id$" - -executablePerms = stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH - - -def S_OK(value=""): - return {"OK": True, "Value": value} - - -def S_ERROR(msg=""): - return {"OK": False, "Message": msg} - - -############ -# Start of CFG -############ - - -class Params(object): - def __init__(self): - self.extensions = [] - self.project = "DIRAC" - self.installation = "DIRAC" - self.release = "" - self.basePath = os.getcwd() - self.targetPath = os.getcwd() - self.debug = False - self.installSource = "" - self.globalDefaults = False - self.timeout = 300 - self.diracOSVersion = "" - self.tag = "" - self.modules = {} - self.createLink = False - self.userEnvVariables = {} - - -cliParams = Params() - -### -# Release config manager -### - - -class ReleaseConfig(object): - class CFG(object): - def __init__(self, cfgData=""): - """c'tor - :param self: self reference - :param str cfgData: the content of the configuration file - """ - self.data = {} - self.children = {} - if cfgData: - self.parse(cfgData) - - def parse(self, cfgData): - """ - It parses the configuration file and propagate the data and children - with the content of the cfg file - :param str cfgData: configuration data, which is the content of the configuration file - """ - try: - self.__parse(cfgData) - except BaseException: - import traceback - - traceback.print_exc() - raise - return self - - def getChild(self, path): - """ - It return the child of a given section - :param str, list, tuple path: for example: Installations/DIRAC, Projects/DIRAC - :return object It returns a CFG instance - """ - - child = self - if isinstance(path, (list, tuple)): - pathList = path - else: - pathList = [sec.strip() for sec in path.split("/") if sec.strip()] - for childName in pathList: - if childName not in child.children: - return False - child = child.children[childName] - return child - - def __parse(self, cfgData, cIndex=0): - """ - It parse a given DIRAC cfg file and store the result in self.data variable. - - :param str cfgData: the content of the configuration file - :param int cIndex: it is the new line counter - """ - - childName = "" - numLine = 0 - while cIndex < len(cfgData): - eol = cfgData.find("\n", cIndex) - if eol < cIndex: - # End? - return cIndex - numLine += 1 - if eol == cIndex: - cIndex += 1 - continue - line = cfgData[cIndex:eol].strip() - # Jump EOL - cIndex = eol + 1 - if not line or line[0] == "#": - continue - if line.find("+=") > -1: - fields = line.split("+=") - opName = fields[0].strip() - if opName in self.data: - self.data[opName] += ", %s" % "+=".join(fields[1:]).strip() - else: - self.data[opName] = "+=".join(fields[1:]).strip() - continue - - if line.find("=") > -1: - fields = line.split("=") - self.data[fields[0].strip()] = "=".join(fields[1:]).strip() - continue - - opFound = line.find("{") - if opFound > -1: - childName += line[:opFound].strip() - if not childName: - raise Exception("No section name defined for opening in line %s" % numLine) - childName = childName.strip() - self.children[childName] = ReleaseConfig.CFG() - eoc = self.children[childName].__parse(cfgData, cIndex) - cIndex = eoc - childName = "" - continue - - if line == "}": - return cIndex - # Must be name for section - childName += line.strip() - return cIndex - - def createSection(self, name, cfg=None): - """ - It creates a subsection for an existing CS section. - :param str name: the name of the section - :param object cfg: the ReleaseConfig.CFG object loaded into memory - """ - - if isinstance(name, (list, tuple)): - pathList = name - else: - pathList = [sec.strip() for sec in name.split("/") if sec.strip()] - parent = self - for lev in pathList[:-1]: - if lev not in parent.children: - parent.children[lev] = ReleaseConfig.CFG() - parent = parent.children[lev] - secName = pathList[-1] - if secName not in parent.children: - if not cfg: - cfg = ReleaseConfig.CFG() - parent.children[secName] = cfg - return parent.children[secName] - - def isSection(self, obList): - """ - Checks if a given path is a section - :param str objList: is a path: for example: Releases/v6r20-pre16 - """ - return self.__exists([ob.strip() for ob in obList.split("/") if ob.strip()]) == 2 - - def sections(self): - """ - Returns all sections - """ - return [k for k in self.children] - - def isOption(self, obList): - return self.__exists([ob.strip() for ob in obList.split("/") if ob.strip()]) == 1 - - def options(self): - """ - Returns the options - """ - return [k for k in self.data] - - def __exists(self, obList): - """ - Check the existence of a certain element - - :param list obList: the list of cfg element names. - for example: [Releases,v6r20-pre16] - """ - if len(obList) == 1: - if obList[0] in self.children: - return 2 - elif obList[0] in self.data: - return 1 - else: - return 0 - if obList[0] in self.children: - return self.children[obList[0]].__exists(obList[1:]) - return 0 - - def get(self, opName, defaultValue=None): - """ - It return the value of a certain option - - :param str opName: the name of the option - :param str defaultValue: the default value of a given option - """ - try: - value = self.__get([op.strip() for op in opName.split("/") if op.strip()]) - except KeyError: - if defaultValue is not None: - return defaultValue - raise - if defaultValue is None: - return value - defType = type(defaultValue) - if isinstance(defType, bool): - return value.lower() in ("1", "true", "yes") - try: - return defType(value) - except ValueError: - return defaultValue - - def __get(self, obList): - """ - It return a given section - - :param list obList: the list of cfg element names. - """ - if len(obList) == 1: - if obList[0] in self.data: - return self.data[obList[0]] - raise KeyError("Missing option %s" % obList[0]) - if obList[0] in self.children: - return self.children[obList[0]].__get(obList[1:]) - raise KeyError("Missing section %s" % obList[0]) - - def toString(self, tabs=0): - """ - It return the configuration file as a string - :param int tabs: the number of tabs used to format the CS string - """ - - lines = ["%s%s = %s" % (" " * tabs, opName, self.data[opName]) for opName in self.data] - for secName in self.children: - lines.append("%s%s" % (" " * tabs, secName)) - lines.append("%s{" % (" " * tabs)) - lines.append(self.children[secName].toString(tabs + 1)) - lines.append("%s}" % (" " * tabs)) - return "\n".join(lines) - - def getOptions(self, path=""): - """ - Rturns the options for a given path - - :param str path: the path to the CS element - """ - parentPath = [sec.strip() for sec in path.split("/") if sec.strip()][:-1] - if parentPath: - parent = self.getChild(parentPath) - else: - parent = self - if not parent: - return [] - return tuple(parent.data) - - def delPath(self, path): - """ - It deletes a given CS element - - :param str path: the path to the CS element - """ - path = [sec.strip() for sec in path.split("/") if sec.strip()] - if not path: - return - keyName = path[-1] - parentPath = path[:-1] - if parentPath: - parent = self.getChild(parentPath) - else: - parent = self - if parent: - parent.data.pop(keyName) - - def update(self, path, cfg): - """ - Used to update the CS - - :param str path: path to the CS element - :param object cfg: the CS object - """ - parent = self.getChild(path) - if not parent: - self.createSection(path, cfg) - return - parent.__apply(cfg) - - def __apply(self, cfg): - """ - It adds a certain cfg subsection to a given section - - :param object cfg: the CS object - """ - for k in cfg.sections(): - if k in self.children: - self.children[k].__apply(cfg.getChild(k)) - else: - self.children[k] = cfg.getChild(k) - for k in cfg.options(): - self.data[k] = cfg.get(k) - - ############################################################################ - # END OF CFG CLASS - ############################################################################ - - def __init__(self, instName="DIRAC", projectName="DIRAC", globalDefaultsURL=None): - """c'tor - :param str instName: the name of the installation - :param str projectName: the name of the project - :param str globalDefaultsURL: the default url - """ - self.user_globalDefaultsURL = globalDefaultsURL - self.globalDefaults = ReleaseConfig.CFG() - self.loadedCfgs = [] - self.prjDepends = {} - self.diracBaseModules = {} - self.prjRelCFG = {} - self.projectsLoadedBy = {} - self.cfgCache = {} - - self.debugCB = False - self.instName = instName - self.projectName = projectName - - def __dbgMsg(self, msg): - """ - :param str msg: the debug message - """ - if self.debugCB: - self.debugCB(msg) - - def __loadCFGFromURL(self, urlcfg, checkHash=False): - """ - It is used to load the configuration file - - :param str urlcfg: the location of the source repository and - where the default configuration file is exists. - :param bool checkHash: check if the file is corrupted. - """ - # This can be a local file - if os.path.exists(urlcfg): - with open(urlcfg, "r") as relFile: - cfgData = relFile.read() - else: - if urlcfg in self.cfgCache: - return S_OK(self.cfgCache[urlcfg]) - try: - cfgData = urlretrieveTimeout(urlcfg, timeout=cliParams.timeout) - if not cfgData: - return S_ERROR("Could not get data from %s" % urlcfg) - except BaseException: - return S_ERROR("Could not open %s" % urlcfg) - try: - # cfgData = cfgFile.read() - cfg = ReleaseConfig.CFG(cfgData) - except Exception as excp: - return S_ERROR("Could not parse %s: %s" % (urlcfg, excp)) - # cfgFile.close() - if not checkHash: - self.cfgCache[urlcfg] = cfg - return S_OK(cfg) - try: - md5path = urlcfg[:-4] + ".md5" - if os.path.exists(md5path): - md5File = open(md5path, "r") - md5Data = md5File.read() - md5File.close() - else: - md5Data = urlretrieveTimeout(md5path, timeout=60) - md5Hex = md5Data.strip() - # md5File.close() - if md5Hex != hashlib.md5(cfgData.encode("utf-8")).hexdigest(): - return S_ERROR("Hash check failed on %s" % urlcfg) - except Exception as excp: - return S_ERROR("Hash check failed on %s: %s" % (urlcfg, excp)) - self.cfgCache[urlcfg] = cfg - return S_OK(cfg) - - def loadInstallationDefaults(self): - """ - Load the default configurations - """ - result = self.__loadGlobalDefaults() - if not result["OK"]: - return result - return self.__loadObjectDefaults("Installations", self.instName) - - def loadProjectDefaults(self): - """ - Load default configurations - """ - result = self.__loadGlobalDefaults() - if not result["OK"]: - return result - return self.__loadObjectDefaults("Projects", self.projectName) - - def __loadGlobalDefaults(self): - """ - It loads the default configuration files - """ - - globalDefaultsURL = self.user_globalDefaultsURL or "/cvmfs/dirac.egi.eu/admin/globalDefaults.cfg" - self.__dbgMsg("Loading global defaults from: %s" % globalDefaultsURL) - result = self.__loadCFGFromURL(globalDefaultsURL) - if not result["OK"]: - default_globalDefaultsURL = "http://diracproject.web.cern.ch/diracproject/configs/globalDefaults.cfg" - self.__dbgMsg("Loading global defaults from: %s" % default_globalDefaultsURL) - result = self.__loadCFGFromURL(default_globalDefaultsURL) - if not result["OK"]: - return result - self.globalDefaults = result["Value"] - for k in ("Installations", "Projects"): - if not self.globalDefaults.isSection(k): - self.globalDefaults.createSection(k) - self.__dbgMsg("Loaded global defaults") - return S_OK() - - def __loadObjectDefaults(self, rootPath, objectName): - """ - It loads the CFG, if it is not loaded. - :param str rootPath: the main section. for example: Installations - :param str objectName: The name of the section. for example: DIRAC - """ - - basePath = "%s/%s" % (rootPath, objectName) - if basePath in self.loadedCfgs: - return S_OK() - - # Check if it's a direct alias - try: - aliasTo = self.globalDefaults.get(basePath) - except KeyError: - aliasTo = False - - if aliasTo: - self.__dbgMsg("%s is an alias to %s" % (objectName, aliasTo)) - result = self.__loadObjectDefaults(rootPath, aliasTo) - if not result["OK"]: - return result - cfg = result["Value"] - self.globalDefaults.update(basePath, cfg) - return S_OK() - - # Load the defaults - if self.globalDefaults.get("%s/SkipDefaults" % basePath, False): - defaultsLocation = "" - else: - defaultsLocation = self.globalDefaults.get("%s/DefaultsLocation" % basePath, "") - - if not defaultsLocation: - self.__dbgMsg("No defaults file defined for %s %s" % (rootPath.lower()[:-1], objectName)) - else: - self.__dbgMsg("Defaults for %s are in %s" % (basePath, defaultsLocation)) - result = self.__loadCFGFromURL(defaultsLocation) - if not result["OK"]: - return result - cfg = result["Value"] - self.globalDefaults.update(basePath, cfg) - - # Check if the defaults have a sub alias - try: - aliasTo = self.globalDefaults.get("%s/Alias" % basePath) - except KeyError: - aliasTo = False - - if aliasTo: - self.__dbgMsg("%s is an alias to %s" % (objectName, aliasTo)) - result = self.__loadObjectDefaults(rootPath, aliasTo) - if not result["OK"]: - return result - cfg = result["Value"] - self.globalDefaults.update(basePath, cfg) - - self.loadedCfgs.append(basePath) - return S_OK(self.globalDefaults.getChild(basePath)) - - def loadInstallationLocalDefaults(self, args): - """ - Load the configuration file from a file - - :param str args: the arguments in which to look for configuration file names - """ - - # at the end we load the local configuration and merge it with the global cfg - argList = list(args) - if os.path.exists("etc/dirac.cfg") and "etc/dirac.cfg" not in args: - argList = ["etc/dirac.cfg"] + argList - - for arg in argList: - if arg.endswith(".cfg") and ":::" not in arg: - fileName = arg - else: - continue - - logNOTICE("Defaults for LocalInstallation are in %s" % fileName) - try: - fd = open(fileName, "r") - cfg = ReleaseConfig.CFG().parse(fd.read()) - fd.close() - except Exception as excp: - logERROR("Could not load %s: %s" % (fileName, excp)) - continue - - self.globalDefaults.update("Installations/%s" % self.instName, cfg) - self.globalDefaults.update("Projects/%s" % self.instName, cfg) - if self.projectName: - # we have an extension and have a local cfg file - self.globalDefaults.update("Projects/%s" % self.projectName, cfg) - - logNOTICE("Loaded %s" % arg) - - def getModuleVersionFromLocalCfg(self, moduleName): - """ - It returns the version of a certain module defined in the LocalInstallation section - :param str moduleName: - :return str: the version of a certain module - """ - return self.globalDefaults.get("Installations/%s/LocalInstallation/%s" % (self.instName, moduleName), "") - - def getInstallationCFG(self, instName=None): - """ - Returns the installation name - - :param str instName: the installation name - """ - if not instName: - instName = self.instName - return self.globalDefaults.getChild("Installations/%s" % instName) - - def getInstallationConfig(self, opName, instName=None): - """ - It returns the configurations from the Installations section. - This is usually provided in the local configuration file - - :param str opName: the option name for example: LocalInstallation/Release - :param str instName: - """ - if not instName: - instName = self.instName - return self.globalDefaults.get("Installations/%s/%s" % (instName, opName)) - - def isProjectLoaded(self, project): - """ - Checks if the project is loaded. - - :param str project: the name of the project - """ - return project in self.prjRelCFG - - def getTarsLocation(self, project, module=None): - """ - Returns the location of the binaries for a given project for example: LHCb or DIRAC, etc... - - :param str project: the name of the project - """ - sourceUrl = self.globalDefaults.get("Projects/%s/BaseURL" % project, "") - if module: - # in case we define a different URL in the CS - differntSourceUrl = self.globalDefaults.get("Projects/%s/%s" % (project, module), "") - if differntSourceUrl: - sourceUrl = differntSourceUrl - if sourceUrl: - return S_OK(sourceUrl) - return S_ERROR("Don't know how to find the installation tarballs for project %s" % project) - - def getDiracOsLocation(self, useVanillaDiracOS=False): - """ - Returns the location of the DIRAC os binary for a given project for example: LHCb or DIRAC, etc... - :param bool useVanillaDiracOS: flag to take diracos distribution from the default location - :return: the location of the tar balls - """ - keysToConsider = [] - if not useVanillaDiracOS: - keysToConsider += [ - "Installations/%s/DIRACOS" % self.projectName, - "Projects/%s/DIRACOS" % self.projectName, - ] - keysToConsider += [ - "Installations/DIRAC/DIRACOS", - "Projects/DIRAC/DIRACOS", - ] - - for key in keysToConsider: - location = self.globalDefaults.get(key, "") - if location: - logDEBUG("Using DIRACOS tarball URL from configuration key %s" % key) - return location - - def __loadReleaseConfig(self, project, release, releaseMode, sourceURL=None, relLocation=None): - """ - It loads the release configuration file - - :param str project: the name of the project - :param str release: the release version - :param str releaseMode: the type of the release server/client - :param str sourceURL: the source of the binary - :param str relLocation: the release configuration file - """ - if project not in self.prjRelCFG: - self.prjRelCFG[project] = {} - if release in self.prjRelCFG[project]: - self.__dbgMsg("Release config for %s:%s has already been loaded" % (project, release)) - return S_OK() - - if relLocation: - relcfgLoc = relLocation - else: - if releaseMode: - try: - relcfgLoc = self.globalDefaults.get("Projects/%s/Releases" % project) - except KeyError: - return S_ERROR("Missing Releases file for project %s" % project) - else: - if not sourceURL: - result = self.getTarsLocation(project) - if not result["OK"]: - return result - siu = result["Value"] - else: - siu = sourceURL - relcfgLoc = "%s/release-%s-%s.cfg" % (siu, project, release) - self.__dbgMsg("Releases file is %s" % relcfgLoc) - result = self.__loadCFGFromURL(relcfgLoc, checkHash=not releaseMode) - if not result["OK"]: - return result - self.prjRelCFG[project][release] = result["Value"] - self.__dbgMsg("Loaded releases file %s" % relcfgLoc) - - return S_OK(self.prjRelCFG[project][release]) - - def getReleaseCFG(self, project, release): - """ - Returns the release configuration object - - :param str project: the name of the project - :param str release: the release version - """ - return self.prjRelCFG[project][release] - - def dumpReleasesToPath(self): - """ - It dumps the content of the loaded configuration (memory content) to - a given file - """ - for project in self.prjRelCFG: - prjRels = self.prjRelCFG[project] - for release in prjRels: - self.__dbgMsg("Dumping releases file for %s:%s" % (project, release)) - fd = open(os.path.join(cliParams.targetPath, "releases-%s-%s.cfg" % (project, release)), "w") - fd.write(prjRels[release].toString()) - fd.close() - - def __checkCircularDependencies(self, key, routePath=None): - """ - Check the dependencies - - :param str key: the name of the project and the release version - :param list routePath: it stores the software packages, used to check the - dependency - """ - - if not routePath: - routePath = [] - if key not in self.projectsLoadedBy: - return S_OK() - routePath.insert(0, key) - for lKey in self.projectsLoadedBy[key]: - if lKey in routePath: - routePath.insert(0, lKey) - route = "->".join(["%s:%s" % sKey for sKey in routePath]) - return S_ERROR("Circular dependency found for %s: %s" % ("%s:%s" % lKey, route)) - result = self.__checkCircularDependencies(lKey, routePath) - if not result["OK"]: - return result - routePath.pop(0) - return S_OK() - - def loadProjectRelease(self, releases, project=None, sourceURL=None, releaseMode=None, relLocation=None): - """ - This method loads all project configurations (*.cfg). If a project is an extension of DIRAC, - it will load the extension and after will load the base DIRAC module. - - :param list releases: list of releases, which will be loaded: for example: v6r19 - :param str project: the name of the project, if it is given. For example: DIRAC - :param str sourceURL: the code repository - :param str releaseMode: - :param str relLocation: local configuration file, - which contains the releases. for example: file:///`pwd`/releases.cfg - """ - - if not project: - project = self.projectName - - if not isinstance(releases, (list, tuple)): - releases = [releases] - - # Load defaults - result = self.__loadObjectDefaults("Projects", project) - if not result["OK"]: - self.__dbgMsg("Could not load defaults for project %s" % project) - return result - - if project not in self.prjDepends: - self.prjDepends[project] = {} - - for release in releases: - self.__dbgMsg("Processing dependencies for %s:%s" % (project, release)) - result = self.__loadReleaseConfig(project, release, releaseMode, sourceURL, relLocation) - if not result["OK"]: - return result - relCFG = result["Value"] - # Calculate dependencies and avoid circular deps - self.prjDepends[project][release] = [(project, release)] - relDeps = self.prjDepends[project][release] - - if not relCFG.getChild("Releases/%s" % (release)): # pylint: disable=no-member - return S_ERROR("Release %s is not defined for project %s in the release file" % (release, project)) - - initialDeps = self.getReleaseDependencies(project, release) - if initialDeps: - self.__dbgMsg( - "%s %s depends on %s" - % (project, release, ", ".join(["%s:%s" % (k, initialDeps[k]) for k in initialDeps])) - ) - relDeps.extend([(p, initialDeps[p]) for p in initialDeps]) - for depProject in initialDeps: - depVersion = initialDeps[depProject] - # Check if already processed - dKey = (depProject, depVersion) - if dKey not in self.projectsLoadedBy: - self.projectsLoadedBy[dKey] = [] - self.projectsLoadedBy[dKey].append((project, release)) - result = self.__checkCircularDependencies(dKey) - if not result["OK"]: - return result - # if it has already been processed just return OK - if len(self.projectsLoadedBy[dKey]) > 1: - return S_OK() - - # Load dependencies and calculate incompatibilities - result = self.loadProjectRelease(depVersion, project=depProject) - if not result["OK"]: - return result - subDep = self.prjDepends[depProject][depVersion] - # Merge dependencies - for sKey in subDep: - if sKey not in relDeps: - relDeps.append(sKey) - continue - prj, vrs = sKey - for pKey in relDeps: - if pKey[0] == prj and pKey[1] != vrs: - errMsg = ( - "%s is required with two different versions ( %s and %s ) \ - starting with %s:%s" - % (prj, pKey[1], vrs, project, release) - ) - return S_ERROR(errMsg) - - # Same version already required - if project in relDeps and relDeps[project] != release: - errMsg = "%s:%s requires itself with a different version through dependencies ( %s )" % ( - project, - release, - relDeps[project], - ) - return S_ERROR(errMsg) - - # we have now all dependencies, let's retrieve the resources (code repository) - for project, version in relDeps: - if project in self.diracBaseModules: - continue - modules = self.getModulesForRelease(version, project) - if modules["OK"]: - for dependency in modules["Value"]: - self.diracBaseModules.setdefault(dependency, {}) - self.diracBaseModules[dependency]["Version"] = modules["Value"][dependency] - res = self.getModSource(version, dependency, project) - if not res["OK"]: - self.__dbgMsg("Unable to found the source URL for %s : %s" % (dependency, res["Message"])) - else: - self.diracBaseModules[dependency]["sourceUrl"] = res["Value"][1] - - return S_OK() - - def getReleaseOption(self, project, release, option): - """ - Returns a given option - - :param str project: the name of the project - :param str release: the release version - :param str option: the option name - """ - try: - return self.prjRelCFG[project][release].get(option) - except KeyError: - self.__dbgMsg("Missing option %s for %s:%s" % (option, project, release)) - # try to found the option in a different release - for project in self.prjRelCFG: - for release in self.prjRelCFG[project]: - if self.prjRelCFG[project][release].isOption(option): - return self.prjRelCFG[project][release].get(option) - return False - - def getReleaseDependencies(self, project, release): - """ - It return the dependencies for a certain project - - :param str project: the name of the project - :param str release: the release version - """ - try: - data = self.prjRelCFG[project][release].get("Releases/%s/Depends" % release) - except KeyError: - return {} - data = [field for field in data.split(",") if field.strip()] - deps = {} - for field in data: - field = field.strip() - if not field: - continue - pv = field.split(":") - if len(pv) == 1: - deps[pv[0].strip()] = release - else: - deps[pv[0].strip()] = ":".join(pv[1:]).strip() - return deps - - def getModulesForRelease(self, release, project=None): - """ - Returns the modules for a given release for example: WebAppDIRAC, - RESTDIRAC, LHCbWebAppDIRAC, etc - - :param str release: the release version - :param str project: the project name - """ - if not project: - project = self.projectName - if project not in self.prjRelCFG: - return S_ERROR("Project %s has not been loaded. I'm a MEGA BUG! Please report me!" % project) - if release not in self.prjRelCFG[project]: - return S_ERROR("Version %s has not been loaded for project %s" % (release, project)) - config = self.prjRelCFG[project][release] - if not config.isSection("Releases/%s" % release): - return S_ERROR("Release %s is not defined for project %s" % (release, project)) - # Defined Modules explicitly in the release - modules = self.getReleaseOption(project, release, "Releases/%s/Modules" % release) - if modules: - dMods = {} - for entry in [ - entry.split(":") for entry in modules.split(",") if entry.strip() - ]: # pylint: disable=no-member - if len(entry) == 1: - dMods[entry[0].strip()] = release - else: - dMods[entry[0].strip()] = entry[1].strip() - modules = dMods - else: - # Default modules with the same version as the release version - modules = self.getReleaseOption(project, release, "DefaultModules") - if modules: - modules = dict( - (modName.strip(), release) for modName in modules.split(",") if modName.strip() - ) # pylint: disable=no-member - else: - # Mod = project and same version - modules = {project: release} - # Check project is in the modNames if not DIRAC - if project != "DIRAC": - for modName in modules: - if modName.find(project) != 0: - return S_ERROR("Module %s does not start with the name %s" % (modName, project)) - return S_OK(modules) - - def getModSource(self, release, modName, project=None): - """ - It reads the Sources section from the .cfg file for example: - Sources - { - Web = git://github.com/DIRACGrid/DIRACWeb.git - VMDIRAC = git://github.com/DIRACGrid/VMDIRAC.git - DIRAC = git://github.com/DIRACGrid/DIRAC.git - BoincDIRAC = git://github.com/DIRACGrid/BoincDIRAC.git - RESTDIRAC = git://github.com/DIRACGrid/RESTDIRAC.git - COMDIRAC = git://github.com/DIRACGrid/COMDIRAC.git - WebAppDIRAC = git://github.com/DIRACGrid/WebAppDIRAC.git - } - - :param str release: the release which is already loaded for example: v6r19 - :param str modName: the name of the DIRAC module for example: WebAppDIRAC - :param str project: the name of the project for example: DIRAC - """ - - if self.projectName not in self.prjRelCFG: - return S_ERROR("Project %s has not been loaded. I'm a MEGA BUG! Please report me!" % self.projectName) - - if not project: - project = self.projectName - modLocation = self.getReleaseOption(project, release, "Sources/%s" % modName) - if not modLocation: - return S_ERROR("Source origin for module %s is not defined" % modName) - modTpl = [field.strip() for field in modLocation.split("|") if field.strip()] # pylint: disable=no-member - if len(modTpl) == 1: - return S_OK((False, modTpl[0])) - return S_OK((modTpl[0], modTpl[1])) - - def getDiracOSExtensionAndVersion(self, diracOSVersion): - """ - This method return the diracos and version taking into - account the extension. The file format will be .tar.gz - - :param str diracOSVersion: column separated string for example: LHCb:v1 - :return: if the extension is not provided, it will return DIRACOS defined in DIRAC otherwise - the DIRACOS specified in the extension - """ - if ":" in diracOSVersion: - package, packageVersion = [i.strip() for i in diracOSVersion.split(":")] - return [package + "diracos", packageVersion] - else: - return ["diracos", diracOSVersion] - - def getDiracOSVersion(self, diracOSVersion=None): - """ - It returns the DIRACOS version - :param str diracOSVersion: the OS version - """ - - if diracOSVersion: - return self.getDiracOSExtensionAndVersion(diracOSVersion) - try: - diracOSVersion = self.prjRelCFG[self.projectName][cliParams.release].get( - "Releases/%s/DIRACOS" % cliParams.release, diracOSVersion - ) - if not diracOSVersion: - # the DIRAC extension does not specify DIRACOS version - for release in self.prjRelCFG["DIRAC"]: - logWARN("Getting DIRACOS version from DIRAC %s!" % release) - diracOSVersion = self.prjRelCFG["DIRAC"][release].get( - "Releases/%s/DIRACOS" % release, diracOSVersion - ) - except KeyError: - pass - return self.getDiracOSExtensionAndVersion(diracOSVersion) - - def getModulesToInstall(self, release, extensions=None): - """ - It returns the modules to be installed. - :param str release: the release version to be deployed - :param str extensions: DIRAC extension - :return: the order of the nodules and modules to be installed. - """ - if not extensions: - extensions = [] - extraFound = [] - modsToInstall = {} - modsOrder = [] - if self.projectName not in self.prjDepends: - return S_ERROR("Project %s has not been loaded" % self.projectName) - if release not in self.prjDepends[self.projectName]: - return S_ERROR("Version %s has not been loaded for project %s" % (release, self.projectName)) - # Get a list of projects with their releases - projects = list(self.prjDepends[self.projectName][release]) - for project, relVersion in projects: - try: - requiredModules = self.prjRelCFG[project][relVersion].get("RequiredExtraModules") - requiredModules = [modName.strip() for modName in requiredModules.split("/") if modName.strip()] - except KeyError: - requiredModules = [] - for modName in requiredModules: - if modName not in extensions: - extensions.append(modName) - self.__dbgMsg("Discovering modules to install for %s (%s)" % (project, relVersion)) - result = self.getModulesForRelease(relVersion, project) - if not result["OK"]: - return result - modVersions = result["Value"] - try: - defaultMods = self.prjRelCFG[project][relVersion].get("DefaultModules") - modNames = [mod.strip() for mod in defaultMods.split(",") if mod.strip()] - except KeyError: - modNames = [] - for extension in extensions: - # Check if the version of the extension module is specified in the command line - extraVersion = None - if ":" in extension: - extension, extraVersion = extension.split(":") - modVersions[extension] = extraVersion - if extension in modVersions: - modNames.append(extension) - extraFound.append(extension) - if "DIRAC" not in extension: - dextension = "%sDIRAC" % extension - if dextension in modVersions: - modNames.append(dextension) - extraFound.append(extension) - modNameVer = ["%s:%s" % (modName, modVersions[modName]) for modName in modNames] - self.__dbgMsg("Modules to be installed for %s are: %s" % (project, ", ".join(modNameVer))) - for modName in modNames: - result = self.getTarsLocation(project, modName) - if not result["OK"]: - return result - tarsURL = result["Value"] - modVersion = modVersions[modName] - defLoc = self.getModuleVersionFromLocalCfg(modName) - if defLoc: - modVersion = defLoc # this overwrite the version which are defined in the release.cfg - modsToInstall[modName] = (tarsURL, modVersion) - modsOrder.insert(0, modName) - - for modName in extensions: - if modName.split(":")[0] not in extraFound: - return S_ERROR("No module %s defined. You sure it's defined for this release?" % modName) - - return S_OK((modsOrder, modsToInstall)) - - -################################################################################# -# End of ReleaseConfig -################################################################################# - - -#### -# Start of helper functions -#### - - -def logDEBUG(msg): - """ - :param str msg: debug message - """ - if cliParams.debug: - for line in msg.split("\n"): - print("%s UTC dirac-install [DEBUG] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()), line)) - sys.stdout.flush() - - -def logERROR(msg): - """ - :param str msg: error message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [ERROR] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()), line)) - sys.stdout.flush() - - -def logWARN(msg): - """ - :param str msg: warning message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [WARN] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()), line)) - sys.stdout.flush() - - -def logNOTICE(msg): - """ - :param str msg: notice message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [NOTICE] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()), line)) - sys.stdout.flush() - - -def alarmTimeoutHandler(*args): - """ - When a connection time out then raise and exception - """ - raise Exception("Timeout") - - -def urlretrieveTimeout(url, fileName="", timeout=0, retries=3): - """ - Retrieve remote url to local file, with timeout wrapper - - :param str fileName: file name - :param int timeout: time out in second used for downloading the files. - """ - if fileName: - # This can be a local file - if os.path.exists(url): # we do not download from web, use locally - logDEBUG('Local file used: "%s"' % url) - shutil.copy(url, fileName) - return True - localFD = open(fileName, "wb") - - # NOTE: Not thread-safe, since all threads will catch same alarm. - # This is OK for dirac-install, since there are no threads. - logDEBUG('Retrieving remote file "%s"' % url) - - urlData = "" - if timeout: - signal.signal(signal.SIGALRM, alarmTimeoutHandler) - # set timeout alarm - signal.alarm(timeout + 5) - try: - # if "http_proxy" in os.environ and os.environ['http_proxy']: - # proxyIP = os.environ['http_proxy'] - # proxy = urllib2.ProxyHandler( {'http': proxyIP} ) - # opener = urllib2.build_opener( proxy ) - # #opener = urllib2.build_opener() - # urllib2.install_opener( opener ) - - # Try to use insecure context explicitly, needed for python >= 2.7.9 - try: - context = ssl._create_unverified_context() - remoteFD = urlopen(url, context=context) # pylint: disable=unexpected-keyword-arg - # the keyword 'context' is present from 2.7.9+ - except AttributeError: - remoteFD = urlopen(url) - - expectedBytes = 0 - # Sometimes repositories do not return Content-Length parameter - try: - expectedBytes = int(remoteFD.info()["Content-Length"]) - except Exception: - logWARN("Content-Length parameter not returned, skipping expectedBytes check") - - receivedBytes = 0 - data = remoteFD.read(16384) - count = 1 - progressBar = False - while data: - receivedBytes += len(data) - if fileName: - localFD.write(data) - else: - urlData += data.decode("utf8", "ignore") - data = remoteFD.read(16384) - if count % 20 == 0 and sys.stdout.isatty(): - print("\033[1D" + ".", end=" ") - sys.stdout.flush() - progressBar = True - count += 1 - if progressBar and sys.stdout.isatty(): - # return cursor to the beginning of the line - print("\033[1K", end=" ") - print("\033[1A") - if fileName: - localFD.close() - remoteFD.close() - if receivedBytes != expectedBytes and expectedBytes > 0: - logERROR("File should be %s bytes but received %s" % (expectedBytes, receivedBytes)) - return False - except HTTPError as x: - if x.code == 404: - logERROR("%s does not exist" % url) - if timeout: - signal.alarm(0) - return False - else: - logWARN("Status code accessing URL %s was %s" % (url, x.getcode())) - if retries: - logWARN("Will retry after 30 seconds, %s retries remaining" % retries) - time.sleep(30) - return urlretrieveTimeout(url, fileName=fileName, timeout=timeout, retries=retries - 1) - else: - logERROR("No retries remaining for %s" % url) - sys.exit(1) - except URLError: - logERROR('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) - except Exception as x: - if x == "Timeout": - logERROR('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) - if timeout: - signal.alarm(0) - raise x - if timeout: - signal.alarm(0) - - if fileName: - return True - else: - return urlData - - -def downloadAndExtractTarball(tarsURL, pkgName, pkgVer, checkHash=True, cache=False): - """ - It downloads and extracts a given tarball from a given destination: file system, - web server or code repository. - - :param str tarsURL: the location of the source repository - :param str pkgName: the name of the package to be installed - :param str pkgVer: the version of the package - :param bool checkHash: check the sanity of the file - :param bool cache: use local cache for the tarballs - - """ - tarName = "%s-%s.tar.gz" % (pkgName, pkgVer) - tarPath = os.path.join(cliParams.targetPath, tarName) - tarFileURL = "%s/%s" % (tarsURL, tarName) - tarFileCVMFS = "/cvmfs/dirac.egi.eu/installSource/%s" % tarName - cacheDir = os.path.join(cliParams.basePath, ".installCache") - tarCachePath = os.path.join(cacheDir, tarName) - isSource = False - if cache and os.path.isfile(tarCachePath): - logNOTICE("Using cached copy of %s" % tarName) - shutil.copy(tarCachePath, tarPath) - elif os.path.exists(tarFileCVMFS): - logNOTICE("Using CVMFS copy of %s" % tarName) - tarPath = tarFileCVMFS - checkHash = False - cache = False - else: - logNOTICE("Retrieving %s" % tarFileURL) - try: - if not urlretrieveTimeout(tarFileURL, tarPath, cliParams.timeout): - if os.path.exists(tarPath): - os.unlink(tarPath) - retVal = checkoutFromGit(pkgName, tarsURL, pkgVer) - if not retVal["OK"]: - logERROR("Cannot download %s" % tarName) - logERROR("Cannot download %s" % retVal["Message"]) - return False - else: - isSource = True - except Exception as e: - logERROR("Cannot download %s: %s" % (tarName, str(e))) - sys.exit(1) - if not isSource and checkHash: - md5Name = "%s-%s.md5" % (pkgName, pkgVer) - md5Path = os.path.join(cliParams.targetPath, md5Name) - md5FileURL = "%s/%s" % (tarsURL, md5Name) - md5CachePath = os.path.join(cacheDir, md5Name) - if cache and os.path.isfile(md5CachePath): - logNOTICE("Using cached copy of %s" % md5Name) - shutil.copy(md5CachePath, md5Path) - else: - logNOTICE("Retrieving %s" % md5FileURL) - try: - if not urlretrieveTimeout(md5FileURL, md5Path, 60): - logERROR("Cannot download %s" % tarName) - return False - except Exception as e: - logERROR("Cannot download %s: %s" % (md5Name, str(e))) - return False - # Read md5 - fd = open(os.path.join(cliParams.targetPath, md5Name), "r") - md5Expected = fd.read().strip() - fd.close() - # Calculate md5 - md5Calculated = hashlib.md5() - with open(os.path.join(cliParams.targetPath, tarName), "rb") as fd: - buf = fd.read(4096) - while buf: - md5Calculated.update(buf) - buf = fd.read(4096) - - # Check - if md5Expected != md5Calculated.hexdigest(): - logERROR("Oops... md5 for package %s failed!" % pkgVer) - sys.exit(1) - # Delete md5 file - if cache: - if not os.path.isdir(cacheDir): - os.makedirs(cacheDir) - os.rename(md5Path, md5CachePath) - else: - os.unlink(md5Path) - # Extract - # cwd = os.getcwd() - # os.chdir(cliParams.targetPath) - # tf = tarfile.open( tarPath, "r" ) - # for member in tf.getmembers(): - # tf.extract( member ) - # os.chdir(cwd) - if not isSource: - logNOTICE("Extract using system tar: %s" % tarPath) - tarCmd = "tar xzf '%s' -C '%s'" % (tarPath, cliParams.targetPath) - if os.system(tarCmd): - logERROR("Extraction of tarball %s failed" % tarPath) - raise RuntimeError("Failed to extract tarball") - # Delete tar - if cache: - if not os.path.isdir(cacheDir): - os.makedirs(cacheDir) - os.rename(tarPath, tarCachePath) - else: - if tarPath != tarFileCVMFS: - os.unlink(tarPath) - - postInstallScript = os.path.join(cliParams.targetPath, pkgName, "dirac-postInstall.py") - if os.path.isfile(postInstallScript): - os.chmod(postInstallScript, executablePerms) - logNOTICE("Executing %s..." % postInstallScript) - if os.system("python '%s' > '%s.out' 2> '%s.err'" % (postInstallScript, postInstallScript, postInstallScript)): - logERROR("Post installation script %s failed. Check %s.err" % (postInstallScript, postInstallScript)) - return True - - -def discoverModules(modules): - """ - Creates the dictionary which contains all modules to install - for example: {"DIRAC:{"sourceUrl":"https://github.com/zmathe/DIRAC.git","Version:v6r20p11"}} - - :param: str modules: it contains meta information for the module, - which will be installed: https://github.com/zmathe/DIRAC.git:::DIRAC:::dev_main_branch - """ - - projects = {} - - for module in modules.split(","): - s = m = v = None - try: - s, m, v = module.split(":::") - except ValueError: - m = module.split(":::")[0] # the source and version is not provided - - projects[m] = {} - if s and v: - projects[m] = {"sourceUrl": s, "Version": v} - else: - logWARN("Unable to parse module: %s" % module) - return projects - - -#### -# End of helper functions -#### - - -cmdOpts = ( - ("r:", "release=", "Release version to install"), - ("l:", "project=", "Project to install"), - ("e:", "extensions=", "Extensions to install (comma separated)"), - ("P:", "installationPath=", "Path where to install (default current working dir)"), - ("u:", "baseURL=", "Use URL as the source for installation tarballs"), - ("d", "debug", "Show debug messages"), - ("V:", "installation=", "Installation from which to extract parameter values"), - ("M:", "defaultsURL=", "Where to retrieve the global defaults from"), - ("h", "help", "Show this help"), - ("T:", "Timeout=", "Timeout for downloads (default = %s)"), - (" ", "dirac-os-version=", "the version of the DIRAC OS"), - (" ", "tag=", "release version to install from git, http or local"), - ("m:", "module=", "Module to be installed. for example: -m DIRAC or -m git://github.com/DIRACGrid/DIRAC.git:DIRAC"), - ( - " ", - "createLink", - "create version symbolic link from the versions directory. This is equivalent to the \ - following command: ln -s /opt/dirac/versions/vArBpC vArBpC", - ), - ( - " ", - "userEnvVariables=", - 'User-requested environment variables (comma-separated, name and value separated by ":::")', - ), -) - - -def usage(): - print("\nUsage:\n\n %s " % os.path.basename(sys.argv[0])) - print("\nOptions:") - for cmdOpt in cmdOpts: - print(" %s %s : %s" % (cmdOpt[0].ljust(3), cmdOpt[1].ljust(20), cmdOpt[2])) - print() - print("Known options and default values from /defaults section of releases file:") - for options in [ - ("Release", cliParams.release), - ("Project", cliParams.project), - ("ModulesToInstall", []), - ("Debug", cliParams.debug), - ("Timeout", cliParams.timeout), - ]: - print(" %s = %s" % options) - - sys.exit(0) - - -def loadConfiguration(): - """ - It loads the configuration file - """ - optList, args = getopt.getopt(sys.argv[1:], "".join([opt[0] for opt in cmdOpts]), [opt[1] for opt in cmdOpts]) - - # First check if the name is defined - for o, v in optList: - if o in ("-h", "--help"): - usage() - elif o in ("-V", "--installation"): - cliParams.installation = v - elif o in ("-d", "--debug"): - cliParams.debug = True - elif o in ("-M", "--defaultsURL"): - cliParams.globalDefaults = v - - releaseConfig = ReleaseConfig(instName=cliParams.installation, globalDefaultsURL=cliParams.globalDefaults) - if cliParams.debug: - releaseConfig.debugCB = logDEBUG - - result = releaseConfig.loadInstallationDefaults() - if not result["OK"]: - logERROR("Could not load defaults: %s" % result["Message"]) - - releaseConfig.loadInstallationLocalDefaults(args) - - for opName in ("release", "debug", "globalDefaults", "targetPath", "project", "release", "extensions", "timeout"): - try: - opVal = releaseConfig.getInstallationConfig("LocalInstallation/%s" % (opName[0].upper() + opName[1:])) - except KeyError: - continue - - if isinstance(getattr(cliParams, opName), str_type): - setattr(cliParams, opName, opVal) - elif isinstance(getattr(cliParams, opName), bool): - setattr(cliParams, opName, opVal.lower() in ("y", "yes", "true", "1")) - elif isinstance(getattr(cliParams, opName), list): - setattr(cliParams, opName, [opV.strip() for opV in opVal.split(",") if opV]) - - # Now parse the ops - for o, v in optList: - if o in ("-r", "--release"): - cliParams.release = v - elif o in ("-l", "--project"): - cliParams.project = v - elif o in ("-e", "--extensions"): - for pkg in [p.strip() for p in v.split(",") if p.strip()]: - if pkg not in cliParams.extensions: - cliParams.extensions.append(pkg) - elif o in ("-d", "--debug"): - cliParams.debug = True - elif o in ("-u", "--baseURL"): - cliParams.installSource = v - elif o in ("-P", "--installationPath"): - cliParams.targetPath = v - try: - os.makedirs(v) - except BaseException: - pass - elif o in ("-T", "--Timeout"): - try: - cliParams.timeout = max(cliParams.timeout, int(v)) - cliParams.timeout = min(cliParams.timeout, 3600) - except ValueError: - pass - elif o == "--dirac-os-version": - cliParams.diracOSVersion = v - elif o == "--tag": - cliParams.tag = v - elif o in ("-m", "--module"): - cliParams.modules = discoverModules(v) - elif o == "--createLink": - cliParams.createLink = True - elif o == "--userEnvVariables": - cliParams.userEnvVariables = dict( - zip( - [name.split(":::")[0] for name in v.replace(" ", "").split(",")], - [value.split(":::")[1] for value in v.replace(" ", "").split(",")], - ) - ) - - if not cliParams.release and not cliParams.modules: - logERROR("Missing release to install") - usage() - - cliParams.basePath = cliParams.targetPath - - logNOTICE("Destination path for installation is %s" % cliParams.targetPath) - releaseConfig.projectName = cliParams.project - - result = releaseConfig.loadProjectRelease( - cliParams.release, project=cliParams.project, sourceURL=cliParams.installSource - ) - if not result["OK"]: - return result - - if not releaseConfig.isProjectLoaded("DIRAC"): - return S_ERROR("DIRAC is not depended by this installation. Aborting") - - # Reload the local configuration to ensure it takes prescience - releaseConfig.loadInstallationLocalDefaults(args) - - return S_OK(releaseConfig) - - -def writeDefaultConfiguration(): - """ - After DIRAC is installed a default configuration file is created, - which contains a minimal setup. - """ - if not releaseConfig: - return - instCFG = releaseConfig.getInstallationCFG() - if not instCFG: - return - for opName in instCFG.getOptions(): - instCFG.delPath(opName) - - # filePath = os.path.join( cliParams.targetPath, "defaults-%s.cfg" % cliParams.installation ) - # Keep the default configuration file in the working directory - filePath = "defaults-%s.cfg" % cliParams.installation - try: - fd = open(filePath, "w") - fd.write(instCFG.toString()) - fd.close() - except Exception as excp: - logERROR("Could not write %s: %s" % (filePath, excp)) - logNOTICE("Defaults written to %s" % filePath) - - -def installDiracOS(releaseConfig): - """ - Install DIRAC OS. - - :param ReleaseConfig releaseConfig: The ReleaseConfig object for configuring the installation - """ - diracos, diracOSVersion = releaseConfig.getDiracOSVersion(cliParams.diracOSVersion) - if not diracOSVersion: - logERROR("No diracos defined") - return False - tarsURL = cliParams.installSource - if not tarsURL: - tarsURL = releaseConfig.getDiracOsLocation(useVanillaDiracOS=(diracos.lower() == "diracos")) - if not tarsURL: - tarsURL = releaseConfig.getTarsLocation("DIRAC")["Value"] - logWARN("DIRACOS location is not specified using %s" % tarsURL) - if not downloadAndExtractTarball(tarsURL, diracos, diracOSVersion, cache=True): - return False - return True - - -def createBashrcForDiracOS(): - """Create DIRAC environment setting script for the bash shell""" - - proPath = cliParams.targetPath - # Now create bashrc at basePath - try: - bashrcFile = os.path.join(cliParams.targetPath, "bashrc") - logNOTICE("Creating %s" % bashrcFile) - if not os.path.exists(bashrcFile): - lines = [ - "# DIRAC bashrc file, used by service and agent run scripts to set environment", - "export PYTHONUNBUFFERED=yes", - "export PYTHONOPTIMIZE=x", - '[ -z "$DIRAC" ] && export DIRAC=%s' % proPath, - '[ -z "$DIRACOS" ] && export DIRACOS=$DIRAC/diracos', - ". $DIRACOS/diracosrc", - ] - if "HOME" in os.environ: - lines.append('[ -z "$HOME" ] && export HOME=%s' % os.environ["HOME"]) - # We also need to ensure LANG is set for py2 unicode support - lines.append('[ -z "$LANG" ] && export LANG=en_US.UTF-8') - - # Helper function for checking dirs - lines.extend( - [ - "# Function check if folder exist and contins files", - "function checkDir () {", - ' if [ -z "${1}" ]; then', - " return 1", - " fi", - ' if [ -n "$(ls -A "${1}" 2>/dev/null)" ]; then', - " return 0", - " fi", - " return 1", - "}", - "", - ] - ) - - # Add sanity check for X509_CERT_DIR variable - lines.extend( - [ - 'if ! checkDir "$X509_CERT_DIR" ; then', - ' export X509_CERT_DIR="/etc/grid-security/certificates"', - ' if ! checkDir "$X509_CERT_DIR" ; then', - ' export X509_CERT_DIR="%s/etc/grid-security/certificates"' % proPath, - " fi", - "fi", - "", - ] - ) - - # Add sanity check for SSL_CERT_DIR variable - lines.extend( - [ - 'if ! checkDir "$SSL_CERT_DIR" ; then', - ' export SSL_CERT_DIR="/etc/grid-security/certificates"', - ' if ! checkDir "$SSL_CERT_DIR" ; then', - ' export SSL_CERT_DIR="%s/etc/grid-security/certificates"' % proPath, - " fi", - "fi", - "", - ] - ) - - # Add sanity check for X509_VOMS_DIR variable - lines.extend( - [ - 'if ! checkDir "$X509_VOMS_DIR" ; then', - ' export X509_VOMS_DIR="%s/etc/grid-security/vomsdir"' % proPath, - "fi", - "", - ] - ) - - # Add sanity check for X509_VOMSES variable - lines.extend( - [ - 'if ! checkDir "$X509_VOMSES" ; then', - ' export X509_VOMSES="%s/etc/grid-security/vomses"' % proPath, - "fi", - "", - ] - ) - - lines.extend(["# Some DIRAC locations", "export DIRACSCRIPTS=%s" % os.path.join("$DIRAC", "scripts")]) - - lines.extend(["# Prepend the PATH and set the PYTHONPATH"]) - - lines.extend(["( echo $PATH | grep -q $DIRACSCRIPTS ) || export PATH=$DIRACSCRIPTS:$PATH"]) - - lines.extend(["export PYTHONPATH=$DIRAC"]) - - lines.extend( - [ - "# new OpenSSL version require OPENSSL_CONF to point to some accessible location", - "export OPENSSL_CONF=/tmp", - ] - ) - - # Add the lines required for globus-* tools to use IPv6 - lines.extend(["# IPv6 support", "export GLOBUS_IO_IPV6=TRUE", "export GLOBUS_FTP_CLIENT_IPV6=TRUE"]) - - # Add the lines required for further env variables requested - if cliParams.userEnvVariables: - lines.extend(["# User-requested variables"]) - for envName, envValue in cliParams.userEnvVariables.items(): - lines.extend(["export %s=%s" % (envName, envValue)]) - - lines.append("") - with open(bashrcFile, "w") as f: - f.write("\n".join(lines)) - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def checkoutFromGit(moduleName, sourceURL, tagVersion, destinationDir=None): - """ - This method checkout a given tag from a git repository. - Note: we can checkout any project form a git repository. - - :param str moduleName: The name of the Module: for example: LHCbWebDIRAC - :param str sourceURL: The code repository: https://github.com/DIRACGrid/WebAppDIRAC.git - :param str tagVersion: the tag for example: v3r1p10 - - """ - - codeRepo = moduleName + "Repo" - - fDirName = os.path.join(cliParams.targetPath, codeRepo) - if os.path.isdir(sourceURL): - logNOTICE("Using local copy for source: %s" % sourceURL) - shutil.copytree(sourceURL, fDirName) - else: - cmd = "git clone '%s' '%s'" % (sourceURL, fDirName) - - logNOTICE("Executing: %s" % cmd) - if os.system(cmd): - return S_ERROR("Error while retrieving sources from git") - - branchName = "%s-%s" % (tagVersion, os.getpid()) - - isTagCmd = "( cd '%s'; git tag -l | grep '%s' )" % (fDirName, tagVersion) - if os.system(isTagCmd): - # No tag found, assume branch - branchSource = "origin/%s" % tagVersion - else: - branchSource = tagVersion - - cmd = "( cd '%s'; git checkout -b '%s' '%s' )" % (fDirName, branchName, branchSource) - - logNOTICE("Executing: %s" % cmd) - exportRes = os.system(cmd) - - if exportRes: - return S_ERROR("Error while exporting from git") - - # replacing the code - if os.path.exists(fDirName + "/" + moduleName): - cmd = "ln -s %s/%s %s" % (codeRepo, moduleName, os.path.join(cliParams.targetPath, moduleName)) - else: - cmd = "mv %s %s" % (fDirName, os.path.join(cliParams.targetPath, moduleName)) - logNOTICE("Executing: %s" % cmd) - retVal = os.system(cmd) - - if retVal: - return S_ERROR("Error while creating module: %s" % (moduleName)) - - return S_OK() - - -def createSymbolicLink(): - """ - It creates a symbolic link to the actual directory from versions - directory. - """ - - cmd = "ln -s %s %s" % (cliParams.targetPath, cliParams.release) - logNOTICE("Executing: %s" % cmd) - retVal = os.system(cmd) - if retVal: - return S_ERROR("Error while creating symbolic link!") - - return S_OK() - - -if __name__ == "__main__": - logNOTICE("Processing installation requirements") - result = loadConfiguration() - releaseConfig = None - modsToInstall = {} - modsOrder = [] - if not result["OK"]: - # the configuration files does not exists, which means the module is not released. - if cliParams.modules: - logNOTICE(str(cliParams.modules)) - for i in cliParams.modules: - modsOrder.append(i) - modsToInstall[i] = (cliParams.modules[i]["sourceUrl"], cliParams.modules[i]["Version"]) - else: - # there is no module provided which can be deployed - logERROR(result["Message"]) - sys.exit(1) - else: - releaseConfig = result["Value"] - - logNOTICE("Discovering modules to install") - if releaseConfig: - result = releaseConfig.getModulesToInstall(cliParams.release, cliParams.extensions) - if not result["OK"]: - logERROR(result["Message"]) - sys.exit(1) - modsOrder, modsToInstall = result["Value"] - if cliParams.debug and releaseConfig: - logNOTICE("Writing down the releases files") - releaseConfig.dumpReleasesToPath() - logNOTICE("Installing modules...") - for modName in set(modsOrder): - tarsURL, modVersion = modsToInstall[modName] - if cliParams.installSource and not cliParams.modules: - # we install not release version of DIRAC - tarsURL = cliParams.installSource - if modName in cliParams.modules: - sourceURL = cliParams.modules[modName].get("sourceUrl") - if "Version" in cliParams.modules[modName]: - modVersion = cliParams.modules[modName]["Version"] - if not sourceURL: - retVal = releaseConfig.getModSource(cliParams.release, modName) - if retVal["OK"]: - tarsURL = retVal["Value"][1] # this is the git repository url - modVersion = cliParams.tag - else: - tarsURL = sourceURL - retVal = checkoutFromGit(modName, tarsURL, modVersion) - if not retVal["OK"]: - logERROR("Cannot checkout %s" % retVal["Message"]) - sys.exit(1) - else: - logNOTICE("Installing %s:%s" % (modName, modVersion)) - if not downloadAndExtractTarball(tarsURL, modName, modVersion): - sys.exit(1) - logNOTICE("Deploying scripts...") - ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "Core", "scripts", "dirac-deploy-scripts.py") - if not os.path.isfile(ddeLocation): - ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "Core", "scripts", "dirac_deploy_scripts.py") - if not os.path.isfile(ddeLocation): - ddeLocation = os.path.join( - cliParams.targetPath, "DIRAC", "src", "DIRAC", "Core", "scripts", "dirac_deploy_scripts.py" - ) - if os.path.isfile(ddeLocation): - cmd = ddeLocation - - if cliParams.modules: - cmd += " " - - os.system(cmd) - else: - logERROR("No dirac-deploy-scripts found. This doesn't look good") - else: - logNOTICE("Skipping installing DIRAC") - - # we install with DIRACOS from v7rX DIRAC release - logNOTICE("Installing DIRAC OS %s..." % cliParams.diracOSVersion) - if not installDiracOS(releaseConfig): - sys.exit(1) - if not createBashrcForDiracOS(): - sys.exit(1) - - writeDefaultConfiguration() - if cliParams.createLink: - createSymbolicLink() - logNOTICE("%s properly installed" % cliParams.installation) - sys.exit(0) diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 91617592..a95ce35c 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -212,7 +212,6 @@ class InstallDIRAC(CommandBase): def __init__(self, pilotParams): """c'tor""" super(InstallDIRAC, self).__init__(pilotParams) - self.installOpts = [] self.pp.rootPath = self.pp.pilotRootPath def _sourceEnvironmentFile(self): @@ -249,62 +248,6 @@ def _saveEnvInFile(self, eFile="environmentSourceDirac"): bl = "export %s=%s\n" % (var, val.rstrip(":")) fd.write(bl) - - def _setInstallOptions(self): - """Setup installation parameters""" - - for o, v in self.pp.optList: - if o == "-d" or o == "--debug": - self.installOpts.append("-d") - elif o == "-e" or o == "--extraPackages": - self.installOpts.append('-e "%s"' % v) - elif o == "-u" or o == "--url": - self.installOpts.append('-u "%s"' % v) - elif o in ("-V", "--installation"): - self.installOpts.append('-V "%s"' % v) - - if self.pp.releaseProject: - self.installOpts.append("-l '%s'" % self.pp.releaseProject) - if self.pp.modules: - self.installOpts.append("-m '%s'" % self.pp.modules) - if self.pp.userEnvVariables: - self.installOpts.append("--userEnvVariables=%s" % self.pp.userEnvVariables) - if self.pp.defaultsURL: - self.installOpts.append("--defaultsURL=%s" % self.pp.defaultsURL) - - # The release version to install is a requirement - self.installOpts.append('-r "%s"' % self.releaseVersion) - - self.log.debug("INSTALL OPTIONS [%s]" % ", ".join(map(str, self.installOpts))) - - def _locateInstallationScript(self): - """Locate installation script""" - installScript = "" - for path in (self.pp.pilotRootPath, self.pp.originalRootPath, self.pp.rootPath): - installScript = os.path.join(path, self.installScriptName) - if os.path.isfile(installScript): - break - self.installScript = installScript - - if not os.path.isfile(installScript): - self.log.error( - "%s requires %s to exist in one of: %s, %s, %s" - % ( - self.pp.pilotScriptName, - self.installScriptName, - self.pp.pilotRootPath, - self.pp.originalRootPath, - self.pp.rootPath, - ) - ) - self.exitWithError(1) - - try: - # change permission of the script - os.chmod(self.installScript, stat.S_IRWXU) - except OSError: - pass - def _getPreinstalledEnvScript(self): """ Get preinstalled environment script if any """ @@ -337,36 +280,10 @@ def _getPreinstalledEnvScript(self): self.pp.preinstalledEnv = preinstalledEnvScript self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript - def _installDIRACpy2(self): - """ Install python2 DIRAC or its extension, - then parse the environment file created, and use it for subsequent calls - Basically, this is used to call dirac-install with the passed parameters. - It requires dirac-install script to be sitting in the same directory. - """ - - # Installing - installCmd = "%s %s" % (self.installScript, " ".join(self.installOpts)) - self.log.debug("Installing with: %s" % installCmd) - - # At this point self.pp.installEnv may coincide with os.environ - # If extensions want to pass in a modified environment, it's easy to set self.pp.installEnv in an extended command - retCode, _output = self.executeAndGetOutput(installCmd, self.pp.installEnv) - - if retCode: - self.log.error("Could not make a proper DIRAC installation [ERROR %d]" % retCode) - self.exitWithError(retCode) - self.log.info("%s completed successfully" % self.installScriptName) - - # Parsing the bashrc then adding its content to the installEnv - # at this point self.pp.installEnv may still coincide with os.environ - self.pp.installEnv["DIRAC_RC_PATH"] = os.path.join(os.getcwd(), "bashrc") - - self._sourceEnvironmentFile() - # At this point self.pp.installEnv should contain all content of bashrc, sourced "on top" of (maybe) os.environ - self._saveEnvInFile() + def _localInstallDIRAC(self): + self.log.info("Installing DIRAC locally") - def _installDIRACpy3(self): """Install python3 version of DIRAC client""" # default to limit the resources used during installation to what the pilot owns installEnv = { @@ -473,17 +390,6 @@ def _installDIRACpy3(self): self.log.error("Could not pip install %s [ERROR %d]" % (self.releaseVersion, retCode)) self.exitWithError(retCode) - def _localInstallDIRAC(self): - self.log.info("Installing DIRAC locally") - if self.pp.pythonVersion == "27": - self.installScriptName = "dirac-install.py" - self.installScript = "" - self._setInstallOptions() - self._locateInstallationScript() - self._installDIRACpy2() - else: # python3 is requested - self._installDIRACpy3() - @logFinalizer def execute(self): """What is called all the time""" @@ -1107,10 +1013,7 @@ def __startJobAgent(self): if self.pp.executeCmd: # Execute user command self.log.info("Executing user defined command: %s" % self.pp.executeCmd) - if self.pp.pythonVersion == "27": - self.exitWithError(int(os.system("source bashrc; %s" % self.pp.executeCmd) / 256)) - else: - self.exitWithError(int(os.system("source diracos/diracosrc; %s" % self.pp.executeCmd) / 256)) + self.exitWithError(int(os.system("source diracos/diracosrc; %s" % self.pp.executeCmd) / 256)) self.log.info("Starting JobAgent") os.environ["PYTHONUNBUFFERED"] = "yes" diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index a12cef61..fc066194 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -92,11 +92,10 @@ def load_module_from_path(module_name, path_to_module): # Utilities functions -def parseVersion(releaseVersion, useLegacyStyle): +def parseVersion(releaseVersion): """Convert the releaseVersion into a legacy or PEP-440 style string :param str releaseVersion: The software version to use - :param bool useLegacyStyle: True to return a vXrY(pZ)(-preN) style version else vX.Y.ZaN """ VERSION_PATTERN = re.compile(r"^(?:v)?(\d+)[r\.](\d+)(?:[p\.](\d+))?(?:(?:-pre|a)?(\d+))?$") @@ -105,17 +104,10 @@ def parseVersion(releaseVersion, useLegacyStyle): if not match: return releaseVersion major, minor, patch, pre = match.groups() - if useLegacyStyle: - version = "v" + major + "r" + minor - if patch and int(patch): - version += "p" + patch - if pre: - version += "-pre" + pre - else: - version = major + "." + minor - version += "." + (patch or "0") - if pre: - version += "a" + pre + version = major + "." + minor + version += "." + (patch or "0") + if pre: + version += "a" + pre return version @@ -840,7 +832,7 @@ def forkAndExecute(self, cmd, logFile, environDict=None): @property def releaseVersion(self): - parsedVersion = parseVersion(self.pp.releaseVersion, self.pp.pythonVersion == "27") + parsedVersion = parseVersion(self.pp.releaseVersion) # strip what is not strictly the version number (e.g. if it is DIRAC[pilot]==7.3.4]) return parsedVersion.split("==")[1] if "==" in parsedVersion else parsedVersion @@ -855,7 +847,6 @@ def __init__(self): """ self.log = Logger(self.__class__.__name__, debugFlag=True) self.rootPath = os.getcwd() - self.originalRootPath = os.getcwd() self.pilotRootPath = os.getcwd() self.workingDir = os.getcwd() @@ -884,7 +875,6 @@ def __init__(self): self.site = "" self.setup = "" self.configServer = "" - self.installation = "" self.ceName = "" self.ceType = "" self.queueName = "" @@ -894,8 +884,6 @@ def __init__(self): # used to set payloadProcessors unless other limits are reached (like the number of processors on the WN) self.maxNumberOfProcessors = 0 self.minDiskSpace = 2560 # MB - self.pythonVersion = "3" - self.defaultsURL = None self.userGroup = "" self.userDN = "" self.maxCycles = 10 @@ -928,8 +916,8 @@ def __init__(self): self.loggerTimerInterval = 0 self.loggerBufsize = 1000 self.pilotUUID = "unknown" - self.modules = "" # see dirac-install "-m" option documentation - self.userEnvVariables = "" # see dirac-install "--userEnvVariables" option documentation + self.modules = "" + self.userEnvVariables = "" self.pipInstallOptions = "" self.CVMFS_locations = [ "/cvmfs/grid.cern.ch", @@ -960,7 +948,7 @@ def __init__(self): ("n:", "name=", "Set as Site Name"), ("o:", "option=", "Option=value to add"), ("m:", "maxNumberOfProcessors=", "specify a max number of processors to use by the payload inside a pilot"), - ("", "modules=", 'for installing non-released code (see dirac-install "-m" option documentation)'), + ("", "modules=", 'for installing non-released code'), ( "", "userEnvVariables=", @@ -992,12 +980,9 @@ def __init__(self): ("R:", "reference=", "Use this pilot reference"), ("S:", "setup=", "DIRAC Setup to use"), ("T:", "CPUTime=", "Requested CPU Time"), - ("V:", "installation=", "Installation configuration file"), ("W:", "gateway=", "Configure as DIRAC Gateway during installation"), ("X:", "commands=", "Pilot commands to execute"), ("Z:", "commandOptions=", "Options parsed by command modules"), - ("", "pythonVersion=", "Python version of DIRAC client to install"), - ("", "defaultsURL=", "user-defined URL for global config"), ("", "pilotUUID=", "pilot UUID"), ("", "preinstalledEnv=", "preinstalled pilot environment script location"), ("", "preinstalledEnvPrefix=", "preinstalled pilot environment area prefix"), @@ -1112,8 +1097,6 @@ def __initCommandLine2(self): self.executeCmd = v elif o in ("-O", "--OwnerDN"): self.userDN = v - elif o in ("-V", "--installation"): - self.installation = v elif o == "-m" or o == "--maxNumberOfProcessors": self.maxNumberOfProcessors = int(v) elif o == "-D" or o == "--disk": @@ -1173,10 +1156,6 @@ def __initCommandLine2(self): self.userEnvVariables = v elif o == "--pipInstallOptions": self.pipInstallOptions = v - elif o == "--pythonVersion": - self.pythonVersion = v - elif o == "--defaultsURL": - self.defaultsURL = v elif o == "--preinstalledEnv": self.preinstalledEnv = v elif o == "--preinstalledEnvPrefix":