Skip to content

Commit

Permalink
Update OfflineGangaDiracSplitter.py
Browse files Browse the repository at this point in the history
  • Loading branch information
mesmith75 authored Jan 11, 2024
1 parent 4e33ed3 commit 25b04ee
Showing 1 changed file with 6 additions and 8 deletions.
14 changes: 6 additions & 8 deletions ganga/GangaDirac/Lib/Splitters/OfflineGangaDiracSplitter.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from GangaCore.Core.exceptions import SplitterError
from GangaDirac.Lib.Backends.DiracUtils import result_ok
from GangaCore.Utility.Config import getConfig
from GangaCore.Utility.logging import getLogger
from GangaDirac.Lib.Utilities.DiracUtilities import execute, GangaDiracError
Expand Down Expand Up @@ -146,7 +145,8 @@ def getLFNReplicas(allLFNs, index, allLFNData):
This method gets the location of all replicas for 'allLFNs' and stores the infomation in 'allLFNData'
This is a 'static' method which is called multiple times with the same 'allLFNs' and 'allLFNData' and different index.
e.g. This allows Dirac to determine the replicas for ~250LFN all at once rather than for ~40,000 all at once which risks timeouts and other errors
e.g. This allows Dirac to determine the replicas for ~250LFN all at once
rather than for ~40,000 all at once which risks timeouts and other errors
Args:
allLFNs (list): This is a list of all LFN which have replicas on the grid
Expand Down Expand Up @@ -242,8 +242,6 @@ def calculateSiteSEMapping(file_replicas, uniqueSE, CE_to_SE_mapping, SE_to_CE_m
"""

SE_dict = dict()
maps_size = 0
found = []

logger.info("Calculating site<->SE Mapping")

Expand Down Expand Up @@ -302,11 +300,13 @@ def calculateSiteSEMapping(file_replicas, uniqueSE, CE_to_SE_mapping, SE_to_CE_m

def lookUpLFNReplicas(inputs, ignoremissing):
"""
This method launches several worker threads to collect the replica information for all LFNs which are given as inputs and stores this in allLFNData
This method launches several worker threads to collect the replica information for all LFNs,
which are given as inputs and stores this in allLFNData
Args:
inputs (list): This is a list of input DiracFile which are
Returns:
bad_lfns (list): A list of LFN which have no replica information when querying `getReplicasForJobs` from DIRAC
bad_lfns (list): A list of LFN which have no replica information when querying
`getReplicasForJobs` from DIRAC
"""
allLFNData = {}
# Build a useful dictionary and list
Expand Down Expand Up @@ -376,8 +376,6 @@ def updateLFNData(bad_lfns, allLFNs, LFNdict, ignoremissing, allLFNData):
if upper_limit > len(allLFNs):
upper_limit = len(allLFNs)

# logger.debug("Updating LFN Physical Locations: [%s:%s] of %s" % (str(i * LFN_parallel_limit), str(upper_limit), str(len(allLFNs))))

for this_lfn in values.keys():
# logger.debug("LFN: %s" % str(this_lfn))
this_dict = {}
Expand Down

0 comments on commit 25b04ee

Please sign in to comment.