Skip to content

Commit

Permalink
Merge pull request #81 from gilles-duboscq/topic/self-reference
Browse files Browse the repository at this point in the history 
Self-references and += operator
  • Loading branch information
@darthbear
darthbear committed May 6, 2016
2 parents 2411739 + fa7ad25 commit f311fba
Show file tree
Hide file tree
Showing 5 changed files with 458 additions and 89 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changelog

# Version 0.3.xx (TBA)

* Implemented self-referential subsitutions

# Version 0.3.25

* ConfigValue.transform: do not wrap lists. PR [#76]
Expand Down
5 changes: 3 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,8 +300,8 @@ Arrays without commas | :white_check_mark:
Path expressions | :white_check_mark:
Paths as keys | :white_check_mark:
Substitutions | :white_check_mark:
Self-referential substitutions | :x:
The `+=` separator | :x:
Self-referential substitutions | :white_check_mark:
The `+=` separator | :white_check_mark:
Includes | :white_check_mark:
Include semantics: merging | :white_check_mark:
Include semantics: substitution | :white_check_mark:
Expand Down Expand Up @@ -335,6 +335,7 @@ Java properties mapping | :x:
- Virgil Palanciuc ([@virgil-palanciuc](https://github.com/virgil-palanciuc))
- Douglas Simon ([@dougxc](https://github.com/dougxc))
- Gilles Duboscq ([@gilles-duboscq](https://github.com/gilles-duboscq))
- Stefan Anzinger ([@sanzinger](https://github.com/sanzinger))

### Thanks

Expand Down
212 changes: 131 additions & 81 deletions pyhocon/config_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import socket
import contextlib
from pyparsing import Forward, Keyword, QuotedString, Word, Literal, Suppress, Regex, Optional, SkipTo, ZeroOrMore, \
Group, lineno, col, TokenConverter, replaceWith, alphanums
Group, lineno, col, TokenConverter, replaceWith, alphanums, ParseSyntaxException
from pyparsing import ParserElement
from pyhocon.config_tree import ConfigTree, ConfigSubstitution, ConfigList, ConfigValues, ConfigUnquotedString, \
ConfigInclude, NoneValue
Expand Down Expand Up @@ -230,27 +230,30 @@ def include_config(token):
(Keyword('url') | Keyword('file')) - Literal('(').suppress() - quoted_string - Literal(')').suppress()))) \
.setParseAction(include_config)

root_dict_expr = Forward()
dict_expr = Forward()
list_expr = Forward()
multi_value_expr = ZeroOrMore((Literal(
'\\') - eol).suppress() | comment_eol | include_expr | substitution_expr | dict_expr | list_expr | value_expr)
# for a dictionary : or = is optional
# last zeroOrMore is because we can have t = {a:4} {b: 6} {c: 7} which is dictionary concatenation
inside_dict_expr = ConfigTreeParser(ZeroOrMore(comment_eol | include_expr | assign_expr | eol_comma))
inside_root_dict_expr = ConfigTreeParser(ZeroOrMore(comment_eol | include_expr | assign_expr | eol_comma), root=True)
dict_expr << Suppress('{') - inside_dict_expr - Suppress('}')
root_dict_expr << Suppress('{') - inside_root_dict_expr - Suppress('}')
list_entry = ConcatenatedValueParser(multi_value_expr)
list_expr << Suppress('[') - ListParser(list_entry - ZeroOrMore(eol_comma - list_entry)) - Suppress(']')

# special case when we have a value assignment where the string can potentially be the remainder of the line
assign_expr << Group(
key -
ZeroOrMore(comment_no_comma_eol) -
(dict_expr | Suppress(Literal('=') | Literal(':')) - ZeroOrMore(
(dict_expr | (Literal('=') | Literal(':') | Literal('+=')) - ZeroOrMore(
comment_no_comma_eol) - ConcatenatedValueParser(multi_value_expr))
)

# the file can be { ... } where {} can be omitted or []
config_expr = ZeroOrMore(comment_eol | eol) + (list_expr | dict_expr | inside_dict_expr) + ZeroOrMore(
config_expr = ZeroOrMore(comment_eol | eol) + (list_expr | root_dict_expr | inside_root_dict_expr) + ZeroOrMore(
comment_eol | eol_comma)
config = config_expr.parseString(content, parseAll=True)[0]
if resolve:
Expand Down Expand Up @@ -290,41 +293,106 @@ def _resolve_variable(config, substitution):
col=col(substitution.loc, substitution.instring)))
return True, value

@staticmethod
def _fixup_self_references(config):
if isinstance(config, ConfigTree) and config.root:
for key in config: # Traverse history of element
history = config.history[key]
previous_item = history[0]
for current_item in history[1:]:
for substitution in ConfigParser._find_substitutions(current_item):
prop_path = ConfigTree.parse_key(substitution.variable)
if len(prop_path) > 1 and config.get(substitution.variable, None) is not None:
continue # If value is present in latest version, don't do anything
if prop_path[0] == key:
if isinstance(previous_item, ConfigValues): # We hit a dead end, we cannot evaluate
raise ConfigSubstitutionException("Property {variable} cannot be substituted. Check for cycles.".format(
variable=substitution.variable))
value = previous_item if len(prop_path) == 1 else previous_item.get(".".join(prop_path[1:]))
(_, _, current_item) = ConfigParser._do_substitute(substitution, value)
previous_item = current_item

if len(history) == 1: # special case, when self optional referencing without existing
for substitution in ConfigParser._find_substitutions(previous_item):
prop_path = ConfigTree.parse_key(substitution.variable)
if len(prop_path) > 1 and config.get(substitution.variable, None) is not None:
continue # If value is present in latest version, don't do anything
if prop_path[0] == key and substitution.optional:
ConfigParser._do_substitute(substitution, None)

# traverse config to find all the substitutions
@staticmethod
def _find_substitutions(item):
"""Convert HOCON input into a JSON output
:return: JSON string representation
:type return: basestring
"""
if isinstance(item, ConfigValues):
return item.get_substitutions()

substitutions = []
if isinstance(item, ConfigTree):
for key, child in item.items():
substitutions += ConfigParser._find_substitutions(child)
elif isinstance(item, list):
for child in item:
substitutions += ConfigParser._find_substitutions(child)
return substitutions

@staticmethod
def _do_substitute(substitution, resolved_value, is_optional_resolved=True):
unresolved = False
new_substitutions = []
if isinstance(resolved_value, ConfigValues):
resolved_value = resolved_value.transform()
if isinstance(resolved_value, ConfigValues):
unresolved = True
result = None
else:
# replace token by substitution
config_values = substitution.parent
# if it is a string, then add the extra ws that was present in the original string after the substitution
formatted_resolved_value = resolved_value \
if resolved_value is None \
or isinstance(resolved_value, (dict, list)) \
or substitution.index == len(config_values.tokens) - 1 \
else (str(resolved_value) + substitution.ws)
config_values.put(substitution.index, formatted_resolved_value)
transformation = config_values.transform()
result = None
if transformation is None and not is_optional_resolved:
result = config_values.overriden_value
else:
result = transformation

if result is None:
del config_values.parent[config_values.key]
else:
config_values.parent[config_values.key] = result
s = ConfigParser._find_substitutions(result)
if s:
new_substitutions = s
unresolved = True

return (unresolved, new_substitutions, result)

@staticmethod
def _final_fixup(item):
if isinstance(item, ConfigValues):
return item.transform()
elif isinstance(item, list):
return list([ConfigParser._final_fixup(child) for child in item])
elif isinstance(item, ConfigTree):
items = list(item.items())
for key, child in items:
item[key] = ConfigParser._final_fixup(child)
return item

@staticmethod
def resolve_substitutions(config):
# traverse config to find all the substitutions
def find_substitutions(item):
"""Convert HOCON input into a JSON output
:return: JSON string representation
:type return: basestring
"""
if isinstance(item, ConfigValues):
return item.get_substitutions()

substitutions = []
if isinstance(item, ConfigTree):
for key, child in item.items():
substitutions += find_substitutions(child)
elif isinstance(item, list):
for child in item:
substitutions += find_substitutions(child)

return substitutions

def final_fixup(item):
if isinstance(item, ConfigValues):
return item.transform()
elif isinstance(item, list):
return list([final_fixup(child) for child in item])
elif isinstance(item, ConfigTree):
items = list(item.items())
for key, child in items:
item[key] = final_fixup(child)

return item

substitutions = find_substitutions(config)
ConfigParser._fixup_self_references(config)
substitutions = ConfigParser._find_substitutions(config)

if len(substitutions) > 0:
unresolved = True
Expand All @@ -340,42 +408,11 @@ def final_fixup(item):
if not is_optional_resolved and substitution.optional:
resolved_value = None

if isinstance(resolved_value, ConfigValues):
resolved_value = resolved_value.transform()
if isinstance(resolved_value, ConfigValues):
unresolved = True
else:
# replace token by substitution
config_values = substitution.parent
# if it is a string, then add the extra ws that was present in the original string after the substitution
formatted_resolved_value = resolved_value \
if resolved_value is None \
or isinstance(resolved_value, (dict, list)) \
or substitution.index == len(config_values.tokens) - 1 \
else (str(resolved_value) + substitution.ws)
config_values.put(substitution.index, formatted_resolved_value)
transformation = config_values.transform()
if transformation is None and not is_optional_resolved:
# if it does not override anything remove the key
# otherwise put back old value that it was overriding
if config_values.overriden_value is None:
if config_values.key in config_values.parent:
del config_values.parent[config_values.key]
else:
config_values.parent[config_values.key] = config_values.overriden_value
s = find_substitutions(config_values.overriden_value)
if s:
substitutions.extend(s)
unresolved = True
else:
config_values.parent[config_values.key] = transformation
s = find_substitutions(transformation)
if s:
substitutions.extend(s)
unresolved = True
substitutions.remove(substitution)

final_fixup(config)
unresolved, new_subsitutions, _ = ConfigParser._do_substitute(substitution, resolved_value, is_optional_resolved)
substitutions.extend(new_subsitutions)
substitutions.remove(substitution)

ConfigParser._final_fixup(config)
if unresolved:
raise ConfigSubstitutionException("Cannot resolve {variables}. Check for cycles.".format(
variables=', '.join('${{{variable}}}: (line: {line}, col: {col})'.format(
Expand Down Expand Up @@ -425,8 +462,9 @@ class ConfigTreeParser(TokenConverter):
Parse a config tree from tokens
"""

def __init__(self, expr=None):
def __init__(self, expr=None, root=False):
super(ConfigTreeParser, self).__init__(expr)
self.root = root
self.saveAsList = True

def postParse(self, instring, loc, token_list):
Expand All @@ -437,35 +475,47 @@ def postParse(self, instring, loc, token_list):
:param token_list:
:return:
"""
config_tree = ConfigTree()
config_tree = ConfigTree(root=self.root)
for element in token_list:
expanded_tokens = element.tokens if isinstance(element, ConfigInclude) else [element]

for tokens in expanded_tokens:
# key, value1 (optional), ...
key = tokens[0].strip()
values = tokens[1:]

operator = '='
if len(tokens) == 3 and tokens[1].strip() in [':', '=', '+=']:
operator = tokens[1].strip()
values = tokens[2:]
elif len(tokens) == 2:
values = tokens[1:]
else:
raise ParseSyntaxException("Unknown tokens {} received".format(tokens))
# empty string
if len(values) == 0:
config_tree.put(key, '')
else:
value = values[0]
if isinstance(value, list):
if isinstance(value, list) and operator == "+=":
value = ConfigValues([ConfigSubstitution(key, True, '', False, loc), value], False, loc)
config_tree.put(key, value, False)
elif isinstance(value, str) and operator == "+=":
value = ConfigValues([ConfigSubstitution(key, True, '', True, loc), ' ' + value], True, loc)
config_tree.put(key, value, False)
elif isinstance(value, list):
config_tree.put(key, value, False)
else:
if isinstance(value, ConfigTree):
existing_value = config_tree.get(key, None)
if isinstance(value, ConfigTree) and not isinstance(existing_value, list):
# Only Tree has to be merged with tree
config_tree.put(key, value, True)
elif isinstance(value, ConfigValues):
conf_value = value
value.parent = config_tree
value.key = key
existing_value = config_tree.get(key, None)
if isinstance(existing_value, list) or isinstance(existing_value, ConfigTree):
config_tree.put(key, conf_value, True)
else:
config_tree.put(key, conf_value, False)
else:
conf_value = value
config_tree.put(key, conf_value, False)
config_tree.put(key, value, False)
return config_tree
Loading

0 comments on commit f311fba

Please sign in to comment.