Skip to content

Commit

Permalink
Removed dict name args to opt for more generic log message when overw…
Browse files Browse the repository at this point in the history
…riting dict values
  • Loading branch information
ca-nguyen committed Aug 19, 2021
1 parent 1dfa0e3 commit 6143783
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 13 deletions.
3 changes: 1 addition & 2 deletions src/stepfunctions/steps/sagemaker.py
Original file line number Diff line number Diff line change
Expand Up @@ -542,8 +542,7 @@ def __init__(self, state_id, processor, job_name, inputs=None, outputs=None, exp

if Field.Parameters.value in kwargs and isinstance(kwargs[Field.Parameters.value], dict):
# Update processing_parameters with input parameters
merge_dicts(processing_parameters, kwargs[Field.Parameters.value], "Processing Parameters",
"Input Parameters")
merge_dicts(processing_parameters, kwargs[Field.Parameters.value])

kwargs[Field.Parameters.value] = processing_parameters
super(ProcessingStep, self).__init__(state_id, **kwargs)
10 changes: 4 additions & 6 deletions src/stepfunctions/steps/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,28 +48,26 @@ def get_aws_partition():
return cur_partition


def merge_dicts(target, source, target_name, source_name):
def merge_dicts(target, source):
"""
Merges source dictionary into the target dictionary.
Values in the target dict are updated with the values of the source dict.
Args:
target (dict): Base dictionary into which source is merged
source (dict): Dictionary used to update target. If the same key is present in both dictionaries, source's value
will overwrite target's value for the corresponding key
target_name (str): Name of target dictionary used for logging purposes
source_name (str): Name of source dictionary used for logging purposes
"""
if isinstance(target, dict) and isinstance(source, dict):
for key, value in source.items():
if key in target:
if isinstance(target[key], dict) and isinstance(source[key], dict):
merge_dicts(target[key], source[key], target_name, source_name)
merge_dicts(target[key], source[key])
elif target[key] == value:
pass
else:
logger.info(
f"{target_name} property: <{key}> with value: <{target[key]}>"
f" will be overwritten with value provided in {source_name} : <{value}>")
f"Property: <{key}> with value: <{target[key]}>"
f" will be overwritten with provided value: <{value}>")
target[key] = source[key]
else:
target[key] = source[key]
38 changes: 33 additions & 5 deletions tests/unit/test_steps_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,36 @@ def test_arn_builder_sagemaker_wait_completion():


def test_merge_dicts():
d1 = {'a': {'aa': 1, 'bb': 2, 'cc': 3}, 'b': 1}
d2 = {'a': {'bb': {'aaa': 1, 'bbb': 2}}, 'b': 2, 'c': 3}

merge_dicts(d1, d2, 'd1', 'd2')
assert d1 == {'a': {'aa': 1, 'bb': {'aaa': 1, 'bbb': 2}, 'cc': 3}, 'b': 2, 'c': 3}
d1 = {
'a': {
'aa': 1,
'bb': 2,
'cc': 3
},
'b': 1
}

d2 = {
'a': {
'bb': {
'aaa': 1,
'bbb': 2
}
},
'b': 2,
'c': 3
}

merge_dicts(d1, d2)
assert d1 == {
'a': {
'aa': 1,
'bb': {
'aaa': 1,
'bbb': 2
},
'cc': 3
},
'b': 2,
'c': 3
}

0 comments on commit 6143783

Please sign in to comment.