Skip to content

Commit

Permalink
convert: Converter pre-processor sceleton.
Browse files Browse the repository at this point in the history
  • Loading branch information
heinezen committed Oct 6, 2019
1 parent 780b164 commit 0f05203
Show file tree
Hide file tree
Showing 18 changed files with 964 additions and 326 deletions.
1 change: 1 addition & 0 deletions openage/convert/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,5 @@ add_subdirectory(dataformat)
add_subdirectory(gamedata)
add_subdirectory(hardcoded)
add_subdirectory(interface)
add_subdirectory(nyan)
add_subdirectory(opus)
3 changes: 2 additions & 1 deletion openage/convert/dataformat/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ add_py_modules(
generated_file.py
header_snippet.py
member_access.py
members.py
multisubtype_base.py
read_members.py
struct_definition.py
struct_snippet.py
util.py
value_member.py
)
188 changes: 188 additions & 0 deletions openage/convert/dataformat/converter_object.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
# Copyright 2014-2019 the openage authors. See copying.md for legal info.

# TODO pylint: disable=C,R,abstract-method

"""
Objects that represent data structures in the original game.
These are simple containers that can be processed by the converter.
"""

from .value_members import ValueMember


class ConverterObject:
"""
Storage object for data objects in the to-be-converted games.
"""

def __init__(self, obj_id, members):
"""
Creates a new ConverterObject.
:param obj_id: An identifier for the object (as a string)
:param members: A list of members.
"""
self.obj_id = obj_id

if all(isinstance(member, ValueMember) for member in members):
self.members = {}

self._create_member_dict(members)

else:
raise Exception("members must be an instance of ValueMember")

def get_id(self):
"""
Returns the object's ID.
"""
return self.obj_id

def add_member(self, member):
"""
Adds a member to the object.
"""
key = member.get_name()
self.members.update({key: member})

def get_member(self, name):
"""
Returns a member of the object.
"""
return self.members[name]

def has_member(self, name):
"""
Returns True if the object has a member with the specified name.
"""
return (name in self.members)

def remove_member(self, name):
"""
Removes a member from the object.
"""
self.members.pop(name)

def short_diff(self, other):
"""
Returns the diff between two objects as another ConverterObject.
The object created by short_diff() only contains members and raw_api_objects
that are different. It does not contain NoDiffMembers.
"""
raise NotImplementedError(
"%s has no short_diff() implementation" % (type(self)))

def diff(self, other):
"""
Returns the diff between two objects as another ConverterObject.
"""
raise NotImplementedError(
"%s has no diff() implementation" % (type(self)))

def _create_member_dict(self, member_list):
"""
Creates the dict from the member list passed to __init__.
"""
for member in member_list:
self.add_member(member)

def __repr__(self):
raise NotImplementedError(
"return short description of the object %s" % (type(self)))


class ConverterObjectGroup:
"""
A group of objects that are connected together in some way
and need each other for conversion. ConverterObjectGroup
instances are converted to the nyan API.
"""

def __init__(self, group_id, raw_api_objects=None):
"""
Creates a new ConverterObjectGroup.
:paran group_id: An identifier for the object group (as a string)
:param raw_api_objects: A list of raw API objects. These will become
proper API objects during conversion.
"""
self.group_id = group_id

# stores the objects that will later be mapped to nyan objects
self.raw_api_objects = {}

if raw_api_objects:
self._create_raw_api_object_dict(raw_api_objects)

def add_raw_api_object(self, subobject):
"""
Adds a subobject to the object.
"""
key = subobject.get_id()
self.raw_api_objects.update({key: subobject})

def get_raw_api_object(self, obj_id):
"""
Returns a subobject of the object.
"""
return self.raw_api_objects[obj_id]

def has_raw_api_object(self, obj_id):
"""
Returns True if the object has a subobject with the specified ID.
"""
return (obj_id in self.raw_api_objects)

def remove_raw_api_object(self, obj_id):
"""
Removes a subobject from the object.
"""
self.raw_api_objects.pop(obj_id)

def _create_raw_api_object_dict(self, subobject_list):
"""
Creates the dict from the subobject list passed to __init__.
"""
for subobject in subobject_list:
self.add_raw_api_object(subobject)

def __repr__(self):
raise NotImplementedError(
"return short description of the object %s" % (type(self)))


class RawAPIObject:
"""
An object that contains all the necessary information to create
a nyan API object.
"""

def __init__(self, api_ref, data):

# fqon of the API object
self.api_ref = api_ref

# A list of ValueMembers that are neceessary to translate
# the object to an actual API object.
self.data = data

def add_data(self, new_data):
"""
Adds more data to the object.
:param new_data: A list of new ValueMembers.
"""
self.data.extend(new_data)

def get_nyan_object(self):
"""
Returns the nyan API object for the raw API object.
"""
raise NotImplementedError(
"returning a nyan object of the object %s not possible" (type(self)))

def __repr__(self):
raise NotImplementedError(
"return short description of the object %s" % (type(self)))
2 changes: 1 addition & 1 deletion openage/convert/dataformat/data_definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from .content_snippet import ContentSnippet, SectionType
from .generated_file import GeneratedFile
from .members import EnumMember, MultisubtypeMember
from openage.convert.dataformat.read_members import EnumMember, MultisubtypeMember
from .util import encode_value, commentify_lines
from .struct_definition import StructDefinition

Expand Down
2 changes: 1 addition & 1 deletion openage/convert/dataformat/data_formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from . import entry_parser
from . import util
from .generated_file import GeneratedFile
from .members import RefMember
from openage.convert.dataformat.read_members import RefMember


class DataFormatter:
Expand Down
51 changes: 32 additions & 19 deletions openage/convert/dataformat/exportable.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
from .data_definition import DataDefinition
from .generated_file import GeneratedFile
from .member_access import READ, READ_EXPORT, READ_UNKNOWN, NOREAD_EXPORT
from .members import (IncludeMembers, ContinueReadMember,
MultisubtypeMember, GroupMember, SubdataMember,
DataMember)
from openage.convert.dataformat.read_members import (IncludeMembers, ContinueReadMember,
MultisubtypeMember, GroupMember, SubdataMember,
ReadMember)
from .struct_definition import (StructDefinition, vararray_match,
integer_match)

Expand Down Expand Up @@ -108,11 +108,13 @@ def dump(self, filename):
"not inheriting from Exportable")

# generate output filename for next-level files
nextlevel_filename = "%s/%04d" % (submember_filename, idx)
nextlevel_filename = "%s/%04d" % (
submember_filename, idx)

# recursive call, fetches DataDefinitions and the
# next-level data dict
data_sets, data = submember_data_item.dump(nextlevel_filename)
data_sets, data = submember_data_item.dump(
nextlevel_filename)

# store recursively generated DataDefinitions to the
# flat list
Expand Down Expand Up @@ -140,7 +142,8 @@ def dump(self, filename):
multisubtype_ref_file_data.append({
MultisubtypeBaseFile.data_format[0][1]: subtype_name,
MultisubtypeBaseFile.data_format[1][1]: "%s%s" % (
subdata_definition.name_data_file, GeneratedFile.output_preferences["csv"]["file_suffix"]
subdata_definition.name_data_file, GeneratedFile.output_preferences[
"csv"]["file_suffix"]
),
})

Expand All @@ -160,7 +163,8 @@ def dump(self, filename):
multisubtype_ref_file = DataDefinition(
MultisubtypeBaseFile,
multisubtype_ref_file_data,
self_data[member_name], # create file to contain refs to subtype files
# create file to contain refs to subtype files
self_data[member_name],
)

subdata_definitions.append(multisubtype_ref_file)
Expand Down Expand Up @@ -195,7 +199,7 @@ def read(self, raw, offset, cls=None, members=None):
for _, export, var_name, var_type in members:

if stop_reading_members:
if isinstance(var_type, DataMember):
if isinstance(var_type, ReadMember):
replacement_value = var_type.get_empty_value()
else:
replacement_value = 0
Expand All @@ -219,7 +223,8 @@ def read(self, raw, offset, cls=None, members=None):
# use its read method to store data to itself,
# then save the result as a reference named `var_name`
# TODO: constructor argument passing may be required here.
grouped_data = var_type.cls(game_versions=self.game_versions)
grouped_data = var_type.cls(
game_versions=self.game_versions)
offset = grouped_data.read(raw, offset)

setattr(self, var_name, grouped_data)
Expand All @@ -235,7 +240,8 @@ def read(self, raw, offset, cls=None, members=None):
if isinstance(var_type.passed_args, str):
var_type.passed_args = set(var_type.passed_args)
for passed_member_name in var_type.passed_args:
varargs[passed_member_name] = getattr(self, passed_member_name)
varargs[passed_member_name] = getattr(
self, passed_member_name)

# subdata list length has to be defined beforehand as a
# object member OR number. it's name or count is specified
Expand All @@ -249,7 +255,8 @@ def read(self, raw, offset, cls=None, members=None):
single_type_subdata = True
else:
# multi-subtype child data list
setattr(self, var_name, {key: [] for key in var_type.class_lookup})
setattr(self, var_name, {key: []
for key in var_type.class_lookup})
single_type_subdata = False

# check if entries need offset checking
Expand Down Expand Up @@ -282,7 +289,8 @@ def read(self, raw, offset, cls=None, members=None):
# read the variable set by the above read call to
# use the read data to determine the denominaton of
# the member type
subtype_name = getattr(self, var_type.subtype_definition[1])
subtype_name = getattr(
self, var_type.subtype_definition[1])

# look up the type name to get the subtype class
new_data_class = var_type.class_lookup[subtype_name]
Expand All @@ -293,7 +301,8 @@ def read(self, raw, offset, cls=None, members=None):
new_data_class.__name__))

# create instance of submember class
new_data = new_data_class(game_versions=self.game_versions, **varargs)
new_data = new_data_class(
game_versions=self.game_versions, **varargs)

# recursive call, read the subdata.
offset = new_data.read(raw, offset, new_data_class)
Expand Down Expand Up @@ -333,23 +342,27 @@ def read(self, raw, offset, cls=None, members=None):
struct_type = var_type
data_count = 1

elif isinstance(var_type, DataMember):
elif isinstance(var_type, ReadMember):
# special type requires having set the raw data type
struct_type = var_type.raw_type
data_count = var_type.get_length(self)
is_custom_member = True

else:
raise Exception("unknown data member definition %s for member '%s'" % (var_type, var_name))
raise Exception(
"unknown data member definition %s for member '%s'" % (var_type, var_name))

if data_count < 0:
raise Exception("invalid length %d < 0 in %s for member '%s'" % (data_count, var_type, var_name))
raise Exception("invalid length %d < 0 in %s for member '%s'" % (
data_count, var_type, var_name))

if struct_type not in struct_type_lookup:
raise Exception("%s: member %s requests unknown data type %s" % (repr(self), var_name, struct_type))
raise Exception("%s: member %s requests unknown data type %s" % (
repr(self), var_name, struct_type))

if export == READ_UNKNOWN:
# for unknown variables, generate uid for the unknown memory location
# for unknown variables, generate uid for the unknown
# memory location
var_name = "unknown-0x%08x" % offset

# lookup c type to python struct scan type
Expand Down Expand Up @@ -464,7 +477,7 @@ def format_hash(cls, hasher=None):
if member_name:
hasher.update(member_name.encode())

if isinstance(member_type, DataMember):
if isinstance(member_type, ReadMember):
hasher = member_type.format_hash(hasher)

elif isinstance(member_type, str):
Expand Down
Loading

0 comments on commit 0f05203

Please sign in to comment.