diff --git a/CHANGES.txt b/CHANGES.txt index 58603e31..f7e826e8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,16 +8,23 @@ All issue numbers are relative to https://github.com/Toblerity/Fiona/issues. Deprecations: -The Python style of rio-filter expressions introduced in version 1.0 are -deprecated. Only the parenthesized list type of expression will be supported by -version 2.0. +- The FIELD_TYPES, FIELD_TYPES_MAP, and FIELD_TYPES_MAP_REV attributes + of fiona.schema are no longer used by the project and will be removed + in version 2.0 (#1366). +- The Python style of rio-filter expressions introduced in version 1.0 are + deprecated. Only the parenthesized list type of expression will be supported + by version 2.0. New features: -The filter, map, and reduce CLI commands from the public domain version 1.1.0 -of fio-planet have been incorporated into Fiona's core set of commands (#1362). -These commands are only available if pyparsing and shapely (each of these are -declared in the "calc" set of extra requirements) are installed. +- All supported Fiona field types are now represented by classes in + fiona.schema. These classes are mapped in FIELD_TYPES_MAP2 and + FIELD_TYPES_MAP2_REV to OGR field type and field subtype pairs + (#1366). +- The filter, map, and reduce CLI commands from the public domain version 1.1.0 + of fio-planet have been incorporated into Fiona's core set of commands + (#1362). These commands are only available if pyparsing and shapely (each of + these are declared in the "calc" set of extra requirements) are installed. Bug fixes: @@ -33,6 +40,11 @@ Bug fixes: - Openers are now registered only by urlpath. The mode is no longer considered as OGR drivers may use a mix of modes when creating a new dataset. +Other changes: + +- Feature builder and field getter/setter instances are reused when + reading and writing features (#1366). + 1.10a1 (2024-03-01) ------------------- diff --git a/fiona/__init__.py b/fiona/__init__.py index db9b0e6b..924137cd 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -46,6 +46,7 @@ get_gdal_version_tuple, ) from fiona._env import driver_count +from fiona._path import _ParsedPath, _UnparsedPath, _parse_path, _vsi_path from fiona._show_versions import show_versions from fiona._vsiopener import _opener_registration from fiona.collection import BytesCollection, Collection @@ -54,15 +55,8 @@ from fiona.errors import FionaDeprecationWarning from fiona.io import MemoryFile from fiona.model import Feature, Geometry, Properties -from fiona.ogrext import ( - FIELD_TYPES_MAP, - _bounds, - _listdir, - _listlayers, - _remove, - _remove_layer, -) -from fiona._path import _ParsedPath, _UnparsedPath, _parse_path, _vsi_path +from fiona.ogrext import _bounds, _listdir, _listlayers, _remove, _remove_layer +from fiona.schema import FIELD_TYPES_MAP, NAMED_FIELD_TYPES from fiona.vfs import parse_paths as vfs_parse_paths # These modules are imported by fiona.ogrext, but are also import here to @@ -534,7 +528,7 @@ def prop_type(text): """ key = text.split(':')[0] - return FIELD_TYPES_MAP[key] + return NAMED_FIELD_TYPES[key].type def drivers(*args, **kwargs): diff --git a/fiona/fio/load.py b/fiona/fio/load.py index 3bb8c48c..a4ba4eff 100644 --- a/fiona/fio/load.py +++ b/fiona/fio/load.py @@ -8,7 +8,6 @@ import fiona from fiona.fio import options, with_context_env from fiona.model import Feature, Geometry -from fiona.schema import FIELD_TYPES_MAP_REV from fiona.transform import transform_geom @@ -87,11 +86,14 @@ def feature_gen(): except TypeError: raise click.ClickException("Invalid input.") - # print(first, first.geometry) + # TODO: this inference of a property's type from its value needs some + # work. It works reliably only for the basic JSON serializable types. + # The fio-load command does require JSON input but that may change + # someday. schema = {"geometry": first.geometry.type} schema["properties"] = { - k: FIELD_TYPES_MAP_REV.get(type(v)) or "str" - for k, v in first.properties.items() + k: type(v if v is not None else "").__name__ + for k, v in first.properties.items() } if append: diff --git a/fiona/gdal.pxi b/fiona/gdal.pxi index 24a51b09..5ed49deb 100644 --- a/fiona/gdal.pxi +++ b/fiona/gdal.pxi @@ -236,7 +236,9 @@ cdef extern from "ogr_core.h" nogil: cdef int OFSTBoolean = 1 cdef int OFSTInt16 = 2 cdef int OFSTFloat32 = 3 - cdef int OFSTMaxSubType = 3 + cdef int OFSTJSON = 4 + cdef int OFSTUUID = 5 + cdef int OFSTMaxSubType = 5 ctypedef struct OGREnvelope: double MinX @@ -553,6 +555,7 @@ cdef extern from "ogr_api.h" nogil: void OGR_F_SetFieldDouble(OGRFeatureH feature, int n, double value) void OGR_F_SetFieldInteger(OGRFeatureH feature, int n, int value) void OGR_F_SetFieldString(OGRFeatureH feature, int n, const char *value) + void OGR_F_SetFieldStringList(OGRFeatureH feature, int n, const char **value) int OGR_F_SetGeometryDirectly(OGRFeatureH feature, OGRGeometryH geometry) OGRFeatureDefnH OGR_FD_Create(const char *name) int OGR_FD_GetFieldCount(OGRFeatureDefnH featuredefn) diff --git a/fiona/ogrext.pyx b/fiona/ogrext.pyx index 629c9fea..604e8c6f 100644 --- a/fiona/ogrext.pyx +++ b/fiona/ogrext.pyx @@ -1,4 +1,4 @@ -# These are extension functions and classes using the OGR C API. +"""Extension classes and functions using the OGR C API.""" include "gdal.pxi" @@ -35,8 +35,7 @@ from fiona.errors import ( from fiona.model import decode_object, Feature, Geometry, Properties from fiona._path import _vsi_path from fiona.rfc3339 import parse_date, parse_datetime, parse_time -from fiona.rfc3339 import FionaDateType, FionaDateTimeType, FionaTimeType -from fiona.schema import FIELD_TYPES, FIELD_TYPES_MAP, normalize_field_type +from fiona.schema import FIELD_TYPES_MAP2, normalize_field_type, NAMED_FIELD_TYPES from libc.stdlib cimport malloc, free from libc.string cimport strcmp @@ -83,15 +82,6 @@ OGRERR_UNSUPPORTED_SRS = 7 OGRERR_INVALID_HANDLE = 8 -cdef bint is_field_null(void *feature, int n): - if OGR_F_IsFieldNull(feature, n): - return True - elif not OGR_F_IsFieldSet(feature, n): - return True - else: - return False - - cdef void gdal_flush_cache(void *cogr_ds): with cpl_errs: GDALFlushCache(cogr_ds) @@ -203,7 +193,6 @@ def _bounds(geometry): except (KeyError, TypeError): return None - cdef int GDAL_VERSION_NUM = get_gdal_version_num() @@ -215,7 +204,350 @@ class TZ(datetime.tzinfo): def utcoffset(self, dt): return datetime.timedelta(minutes=self.minutes) -# Feature extension classes and functions follow. + +cdef class AbstractField: + + cdef object driver + cdef object supports_tz + + def __init__(self, driver=None): + self.driver = driver + + cdef object name(self, OGRFieldDefnH fdefn): + """Get the short Fiona field name corresponding to the OGR field definition.""" + raise NotImplementedError + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + """Get the value of a feature's field.""" + raise NotImplementedError + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + """Set the value of a feature's field.""" + raise NotImplementedError + + +cdef class IntegerField(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + cdef int width = OGR_Fld_GetWidth(fdefn) + fmt = "" + if width: + fmt = f":{width:d}" + return f"int32{fmt}" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + return OGR_F_GetFieldAsInteger(feature, i) + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + OGR_F_SetFieldInteger(feature, i, int(value)) + + +cdef class Int16Field(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + cdef int width = OGR_Fld_GetWidth(fdefn) + fmt = "" + if width: + fmt = f":{width:d}" + return f"int16{fmt}" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + return OGR_F_GetFieldAsInteger(feature, i) + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + OGR_F_SetFieldInteger(feature, i, int(value)) + + +cdef class BooleanField(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + return "bool" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + return bool(OGR_F_GetFieldAsInteger(feature, i)) + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + OGR_F_SetFieldInteger(feature, i, int(value)) + + +cdef class Integer64Field(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + cdef int width = OGR_Fld_GetWidth(fdefn) + fmt = "" + if width: + fmt = f":{width:d}" + return f"int{fmt}" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + return OGR_F_GetFieldAsInteger64(feature, i) + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + OGR_F_SetFieldInteger64(feature, i, int(value)) + + +cdef class RealField(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + cdef int width = OGR_Fld_GetWidth(fdefn) + cdef int precision = OGR_Fld_GetPrecision(fdefn) + fmt = "" + if width: + fmt = f":{width:d}" + if precision: + fmt += f".{precision:d}" + return f"float{fmt}" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + return OGR_F_GetFieldAsDouble(feature, i) + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + OGR_F_SetFieldDouble(feature, i, float(value)) + + +cdef class StringField(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + cdef int width = OGR_Fld_GetWidth(fdefn) + fmt = "" + if width: + fmt = f":{width:d}" + return f"str{fmt}" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + encoding = kwds["encoding"] + val = OGR_F_GetFieldAsString(feature, i) + try: + val = val.decode(encoding) + except UnicodeDecodeError: + log.warning( + "Failed to decode %s using %s codec", val, encoding) + else: + return val + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + encoding = kwds["encoding"] + cdef object value_b = str(value).encode(encoding) + OGR_F_SetFieldString(feature, i, value_b) + + +cdef class BinaryField(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + return "bytes" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + cdef unsigned char *data = NULL + cdef int l + data = OGR_F_GetFieldAsBinary(feature, i, &l) + return data[:l] + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + OGR_F_SetFieldBinary(feature, i, len(value), value) + + +cdef class StringListField(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + return "List[str]" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + cdef char **string_list = NULL + encoding = kwds["encoding"] + string_list = OGR_F_GetFieldAsStringList(feature, i) + string_list_index = 0 + vals = [] + if string_list != NULL: + while string_list[string_list_index] != NULL: + val = string_list[string_list_index] + try: + val = val.decode(encoding) + except UnicodeDecodeError: + log.warning( + "Failed to decode %s using %s codec", val, encoding + ) + vals.append(val) + string_list_index += 1 + return vals + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + cdef char **string_list = NULL + encoding = kwds["encoding"] + for item in value: + item_b = item.encode(encoding) + string_list = CSLAddString(string_list, item_b) + OGR_F_SetFieldStringList(feature, i, string_list) + + +cdef class JSONField(AbstractField): + + cdef object name(self, OGRFieldDefnH fdefn): + return "json" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + val = OGR_F_GetFieldAsString(feature, i) + return json.loads(val) + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + value_b = json.dumps(value).encode("utf-8") + OGR_F_SetFieldString(feature, i, value_b) + + +cdef class DateField(AbstractField): + """Dates without time.""" + + cdef object name(self, OGRFieldDefnH fdefn): + return "date" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + cdef int retval + cdef int y = 0 + cdef int m = 0 + cdef int d = 0 + cdef int hh = 0 + cdef int mm = 0 + cdef float fss = 0.0 + cdef int tz = 0 + retval = OGR_F_GetFieldAsDateTimeEx(feature, i, &y, &m, &d, &hh, &mm, &fss, &tz) + return datetime.date(y, m, d).isoformat() + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + if isinstance(value, str): + y, m, d, hh, mm, ss, ms, tz = parse_date(value) + elif isinstance(value, datetime.date): + y, m, d = value.year, value.month, value.day + hh = mm = ss = ms = 0 + else: + raise ValueError() + + tzinfo = 0 + OGR_F_SetFieldDateTimeEx(feature, i, y, m, d, hh, mm, ss, tzinfo) + + +cdef class TimeField(AbstractField): + """Times without dates.""" + + def __init__(self, driver=None): + self.driver = driver + self.supports_tz = _driver_supports_timezones(self.driver, "time") + + cdef object name(self, OGRFieldDefnH fdefn): + return "time" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + cdef int retval + cdef int y = 0 + cdef int m = 0 + cdef int d = 0 + cdef int hh = 0 + cdef int mm = 0 + cdef float fss = 0.0 + cdef int tz = 0 + retval = OGR_F_GetFieldAsDateTimeEx(feature, i, &y, &m, &d, &hh, &mm, &fss, &tz) + ms, ss = math.modf(fss) + ss = int(ss) + ms = int(round(ms * 10**6)) + # OGR_F_GetFieldAsDateTimeEx: (0=unknown, 1=localtime, 100=GMT, see data model for details) + # CPLParseRFC822DateTime: (0=unknown, 100=GMT, 101=GMT+15minute, 99=GMT-15minute), or NULL + tzinfo = None + if tz > 1: + tz_minutes = (tz - 100) * 15 + tzinfo = TZ(tz_minutes) + return datetime.time(hh, mm, ss, ms, tzinfo).isoformat() + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + if isinstance(value, str): + y, m, d, hh, mm, ss, ms, tz = parse_time(value) + elif isinstance(value, datetime.time): + y = m = d = 0 + hh, mm, ss, ms = value.hour, value.minute, value.second, value.microsecond + if value.utcoffset() is None: + tz = None + else: + tz = value.utcoffset().total_seconds() / 60 + else: + raise ValueError() + + if tz is not None and not self.supports_tz: + d_tz = datetime.datetime(1900, 1, 1, hh, mm, ss, int(ms), TZ(tz)) + d_utc = d_tz - d_tz.utcoffset() + y = m = d = 0 + hh, mm, ss, ms = d_utc.hour, d_utc.minute, d_utc.second, d_utc.microsecond + tz = 0 + + # tzinfo: (0=unknown, 100=GMT, 101=GMT+15minute, 99=GMT-15minute), or NULL + if tz is not None: + tzinfo = int(tz / 15.0 + 100) + else: + tzinfo = 0 + + ss += ms / 10**6 + OGR_F_SetFieldDateTimeEx(feature, i, y, m, d, hh, mm, ss, tzinfo) + + +cdef class DateTimeField(AbstractField): + """Dates and times.""" + + def __init__(self, driver=None): + self.driver = driver + self.supports_tz = _driver_supports_timezones(self.driver, "datetime") + + cdef object name(self, OGRFieldDefnH fdefn): + return "datetime" + + cdef object get(self, OGRFeatureH feature, int i, object kwds): + cdef int retval + cdef int y = 0 + cdef int m = 0 + cdef int d = 0 + cdef int hh = 0 + cdef int mm = 0 + cdef float fss = 0.0 + cdef int tz = 0 + retval = OGR_F_GetFieldAsDateTimeEx(feature, i, &y, &m, &d, &hh, &mm, &fss, &tz) + ms, ss = math.modf(fss) + ss = int(ss) + ms = int(round(ms * 10**6)) + # OGR_F_GetFieldAsDateTimeEx: (0=unknown, 1=localtime, 100=GMT, see data model for details) + # CPLParseRFC822DateTime: (0=unknown, 100=GMT, 101=GMT+15minute, 99=GMT-15minute), or NULL + tzinfo = None + if tz > 1: + tz_minutes = (tz - 100) * 15 + tzinfo = TZ(tz_minutes) + return datetime.datetime(y, m, d, hh, mm, ss, ms, tzinfo).isoformat() + + cdef set(self, OGRFeatureH feature, int i, object value, object kwds): + if isinstance(value, str): + y, m, d, hh, mm, ss, ms, tz = parse_datetime(value) + elif isinstance(value, datetime.datetime): + y, m, d = value.year, value.month, value.day + hh, mm, ss, ms = value.hour, value.minute, value.second, value.microsecond + if value.utcoffset() is None: + tz = None + else: + tz = value.utcoffset().total_seconds() / 60 + else: + raise ValueError() + + if tz is not None and not self.supports_tz: + d_tz = datetime.datetime(y, m, d, hh, mm, ss, int(ms), TZ(tz)) + d_utc = d_tz - d_tz.utcoffset() + y, m, d = d_utc.year, d_utc.month, d_utc.day + hh, mm, ss, ms = d_utc.hour, d_utc.minute, d_utc.second, d_utc.microsecond + tz = 0 + + # tzinfo: (0=unknown, 100=GMT, 101=GMT+15minute, 99=GMT-15minute), or NULL + if tz is not None: + tzinfo = int(tz / 15.0 + 100) + else: + tzinfo = 0 + + ss += ms / 10**6 + OGR_F_SetFieldDateTimeEx(feature, i, y, m, d, hh, mm, ss, tzinfo) + + +cdef bint is_field_null(OGRFeatureH feature, int i): + return OGR_F_IsFieldNull(feature, i) or not OGR_F_IsFieldSet(feature, i) cdef class FeatureBuilder: @@ -225,7 +557,37 @@ cdef class FeatureBuilder: argument is not destroyed. """ - cdef build(self, void *feature, encoding='utf-8', bbox=False, driver=None, ignore_fields=None, ignore_geometry=False): + cdef object driver + cdef object property_getter_cache + + OGRPropertyGetter = { + (OFTInteger, OFSTNone): IntegerField, + (OFTInteger, OFSTBoolean): BooleanField, + (OFTInteger, OFSTInt16): Int16Field, + (OFTInteger64, OFSTNone): Integer64Field, + (OFTReal, OFSTNone): RealField, + (OFTString, OFSTNone): StringField, + (OFTDate, OFSTNone): DateField, + (OFTTime, OFSTNone): TimeField, + (OFTDateTime, OFSTNone): DateTimeField, + (OFTBinary, OFSTNone): BinaryField, + (OFTStringList, OFSTNone): StringListField, + (OFTString, OFSTJSON): JSONField, + } + + def __init__(self, driver=None): + self.driver = driver + self.property_getter_cache = {} + + cdef build( + self, + OGRFeatureH feature, + encoding='utf-8', + bbox=False, + driver=None, + ignore_fields=None, + ignore_geometry=False + ): """Build a Fiona feature object from an OGR feature Parameters @@ -248,23 +610,12 @@ cdef class FeatureBuilder: ------- dict """ - cdef void *fdefn = NULL + cdef OGRFieldDefnH fdefn cdef int i - cdef unsigned char *data = NULL - cdef char **string_list = NULL - cdef int string_list_index = 0 - cdef int l - cdef int retval + cdef int fieldtype cdef int fieldsubtype - cdef const char *key_c = NULL - # Parameters for OGR_F_GetFieldAsDateTimeEx - cdef int y = 0 - cdef int m = 0 - cdef int d = 0 - cdef int hh = 0 - cdef int mm = 0 - cdef float fss = 0.0 - cdef int tz = 0 + cdef const char *key_c + cdef AbstractField getter # Skeleton of the feature to be returned. fid = OGR_F_GetFID(feature) @@ -272,118 +623,52 @@ cdef class FeatureBuilder: ignore_fields = set(ignore_fields or []) - # Iterate over the fields of the OGR feature. for i in range(OGR_F_GetFieldCount(feature)): fdefn = OGR_F_GetFieldDefnRef(feature, i) if fdefn == NULL: raise ValueError(f"NULL field definition at index {i}") + key_c = OGR_Fld_GetNameRef(fdefn) if key_c == NULL: raise ValueError(f"NULL field name reference at index {i}") + key_b = key_c key = key_b.decode(encoding) + + # Some field names are empty strings, apparently. + # We warn in this case. if not key: warnings.warn(f"Empty field name at index {i}") if key in ignore_fields: continue - fieldtypename = FIELD_TYPES[OGR_Fld_GetType(fdefn)] + fieldtype = OGR_Fld_GetType(fdefn) fieldsubtype = OGR_Fld_GetSubType(fdefn) - if not fieldtypename: - log.warning( - "Skipping field %s: invalid type %s", - key, - OGR_Fld_GetType(fdefn)) - continue - - # TODO: other types - fieldtype = FIELD_TYPES_MAP[fieldtypename] + fieldkey = (fieldtype, fieldsubtype) if is_field_null(feature, i): props[key] = None - - elif fieldtypename is 'int32': - if fieldsubtype == OFSTBoolean: - props[key] = bool(OGR_F_GetFieldAsInteger(feature, i)) - else: - props[key] = OGR_F_GetFieldAsInteger(feature, i) - - elif fieldtype is int: - if fieldsubtype == OFSTBoolean: - props[key] = bool(OGR_F_GetFieldAsInteger64(feature, i)) + else: + if fieldkey in self.property_getter_cache: + getter = self.property_getter_cache[fieldkey] else: - props[key] = OGR_F_GetFieldAsInteger64(feature, i) - - elif fieldtype is float: - props[key] = OGR_F_GetFieldAsDouble(feature, i) - - elif fieldtype is str: - val = OGR_F_GetFieldAsString(feature, i) - try: - val = val.decode(encoding) - except UnicodeDecodeError: - log.warning( - "Failed to decode %s using %s codec", val, encoding) - - # Does the text contain a JSON object? Let's check. - # Let's check as cheaply as we can. - if driver == 'GeoJSON' and val.startswith('{'): try: - val = json.loads(val) - except ValueError as err: - log.warning(str(err)) - - # Now add to the properties object. - props[key] = val - - elif fieldtype in (FionaDateType, FionaTimeType, FionaDateTimeType): - retval = OGR_F_GetFieldAsDateTimeEx(feature, i, &y, &m, &d, &hh, &mm, &fss, &tz) - ms, ss = math.modf(fss) - ss = int(ss) - ms = int(round(ms * 10**6)) - - # OGR_F_GetFieldAsDateTimeEx: (0=unknown, 1=localtime, 100=GMT, see data model for details) - # CPLParseRFC822DateTime: (0=unknown, 100=GMT, 101=GMT+15minute, 99=GMT-15minute), or NULL - tzinfo = None - if tz > 1: - tz_minutes = (tz - 100) * 15 - tzinfo = TZ(tz_minutes) + getter = self.OGRPropertyGetter[fieldkey](driver=driver or self.driver) + self.property_getter_cache[fieldkey] = getter + except KeyError: + log.warning( + "Skipping field %s: invalid type %s", + key, + fieldkey + ) + continue - try: - if fieldtype is FionaDateType: - props[key] = datetime.date(y, m, d).isoformat() - elif fieldtype is FionaTimeType: - props[key] = datetime.time(hh, mm, ss, ms, tzinfo).isoformat() - else: - props[key] = datetime.datetime(y, m, d, hh, mm, ss, ms, tzinfo).isoformat() - except ValueError as err: - log.exception(err) - props[key] = None - - elif fieldtype is bytes: - data = OGR_F_GetFieldAsBinary(feature, i, &l) - props[key] = data[:l] - elif fieldtype is List[str]: - string_list = OGR_F_GetFieldAsStringList(feature, i) - string_list_index = 0 - props[key] = [] - if string_list != NULL: - while string_list[string_list_index] != NULL: - val = string_list[string_list_index] - try: - val = val.decode(encoding) - except UnicodeDecodeError: - log.warning( - "Failed to decode %s using %s codec", val, encoding - ) - props[key].append(val) - string_list_index += 1 - else: - props[key] = None + props[key] = getter.get(feature, i, {"encoding": encoding}) cdef void *cogr_geometry = NULL geom = None + if not ignore_geometry: cogr_geometry = OGR_F_GetGeometryRef(feature) geom = GeomBuilder().build_from_feature(feature) @@ -399,13 +684,56 @@ cdef class OGRFeatureBuilder: Borrows a layer definition from the collection. """ - cdef void * build(self, feature, collection) except NULL: + + cdef object driver + cdef object property_setter_cache + + OGRPropertySetter = { + (OFTInteger, OFSTNone, "int"): IntegerField, + (OFTInteger, OFSTNone, "int32"): IntegerField, + (OFTInteger, OFSTNone, "float"): RealField, + (OFTInteger, OFSTNone, "str"): StringField, + (OFTInteger, OFSTBoolean, "bool"): BooleanField, + (OFTInteger, OFSTBoolean, "int"): BooleanField, + (OFTInteger, OFSTInt16, "int"): Int16Field, + (OFTInteger, OFSTInt16, "str"): StringField, + (OFTInteger64, OFSTNone, "int"): Integer64Field, + (OFTInteger64, OFSTNone, "int64"): Integer64Field, + (OFTInteger64, OFSTNone, "float"): RealField, + (OFTInteger64, OFSTNone, "str"): StringField, + (OFTReal, OFSTNone, "float"): RealField, + (OFTReal, OFSTNone, "str"): StringField, + (OFTReal, OFSTFloat32, "float"): RealField, + (OFTReal, OFSTFloat32, "float32"): RealField, + (OFTReal, OFSTFloat32, "str"): StringField, + (OFTString, OFSTNone, "str"): StringField, + (OFTString, OFSTNone, "dict"): StringField, + (OFTDate, OFSTNone, "date"): DateField, + (OFTDate, OFSTNone, "str"): DateField, + (OFTTime, OFSTNone, "time"): TimeField, + (OFTTime, OFSTNone, "str"): TimeField, + (OFTDateTime, OFSTNone, "datetime"): DateTimeField, + (OFTDateTime, OFSTNone, "str"): DateTimeField, + (OFTBinary, OFSTNone, "bytes"): BinaryField, + (OFTBinary, OFSTNone, "bytearray"): BinaryField, + (OFTBinary, OFSTNone, "memoryview"): BinaryField, + (OFTStringList, OFSTNone, "list"): StringListField, + (OFTString, OFSTJSON, "dict"): JSONField, + (OFTString, OFSTJSON, "list"): JSONField, + } + + def __init__(self, driver=None): + self.driver = driver + self.property_setter_cache = {} + + cdef OGRFeatureH build(self, feature, collection) except NULL: cdef void *cogr_geometry = NULL cdef const char *string_c = NULL cdef WritingSession session = collection.session cdef void *cogr_layer = session.cogr_layer cdef void *cogr_featuredefn = OGR_L_GetLayerDefn(cogr_layer) cdef void *cogr_feature = OGR_F_Create(cogr_featuredefn) + cdef AbstractField setter if cogr_layer == NULL: raise ValueError("Null layer") @@ -420,7 +748,6 @@ cdef class OGRFeatureBuilder: cogr_geometry = OGRGeomBuilder().build(feature.geometry) exc_wrap_int(OGR_F_SetGeometryDirectly(cogr_feature, cogr_geometry)) - # OGR_F_SetFieldString takes encoded strings ('bytes' in Python 3). encoding = session._get_internal_encoding() for key, value in feature.properties.items(): @@ -429,89 +756,32 @@ cdef class OGRFeatureBuilder: if i < 0: continue - schema_type = session._schema_normalized_field_types[key] - - # Special case: serialize dicts to assist OGR. - if isinstance(value, dict): - value = json.dumps(value) - - # Continue over the standard OGR types. - if isinstance(value, int): - if schema_type == 'int32': - OGR_F_SetFieldInteger(cogr_feature, i, value) - else: - OGR_F_SetFieldInteger64(cogr_feature, i, value) - - elif isinstance(value, float): - OGR_F_SetFieldDouble(cogr_feature, i, value) - elif schema_type in ['date', 'time', 'datetime'] and value is not None: - if isinstance(value, str): - if schema_type == 'date': - y, m, d, hh, mm, ss, ms, tz = parse_date(value) - elif schema_type == 'time': - y, m, d, hh, mm, ss, ms, tz = parse_time(value) - else: - y, m, d, hh, mm, ss, ms, tz = parse_datetime(value) - elif (isinstance(value, datetime.date) and schema_type == 'date'): - y, m, d = value.year, value.month, value.day - hh = mm = ss = ms = 0 - tz = None - elif (isinstance(value, datetime.datetime) and schema_type == 'datetime'): - y, m, d = value.year, value.month, value.day - hh, mm, ss, ms = value.hour, value.minute, value.second, value.microsecond - if value.utcoffset() is None: - tz = None - else: - tz = value.utcoffset().total_seconds() / 60 - elif (isinstance(value, datetime.time) and schema_type == 'time'): - y = m = d = 0 - hh, mm, ss, ms = value.hour, value.minute, value.second, value.microsecond - if value.utcoffset() is None: - tz = None - else: - tz = value.utcoffset().total_seconds() / 60 - - # Convert to UTC if driver does not support timezones - if tz is not None and not _driver_supports_timezones(collection.driver, schema_type): - - if schema_type == 'datetime': - d_tz = datetime.datetime(y, m, d, hh, mm, ss, int(ms), TZ(tz)) - d_utc = d_tz - d_tz.utcoffset() - y, m, d = d_utc.year, d_utc.month, d_utc.day - hh, mm, ss, ms = d_utc.hour, d_utc.minute, d_utc.second, d_utc.microsecond - tz = 0 - del d_utc, d_tz - elif schema_type == 'time': - d_tz = datetime.datetime(1900, 1, 1, hh, mm, ss, int(ms), TZ(tz)) - d_utc = d_tz - d_tz.utcoffset() - y = m = d = 0 - hh, mm, ss, ms = d_utc.hour, d_utc.minute, d_utc.second, d_utc.microsecond - tz = 0 - del d_utc, d_tz - - # tzinfo: (0=unknown, 100=GMT, 101=GMT+15minute, 99=GMT-15minute), or NULL - if tz is not None: - tzinfo = int(tz / 15.0 + 100) - else: - tzinfo = 0 - - # Add microseconds to seconds - ss += ms / 10**6 - - OGR_F_SetFieldDateTimeEx(cogr_feature, i, y, m, d, hh, mm, ss, tzinfo) - - elif isinstance(value, bytes) and schema_type == "bytes": - string_c = value - OGR_F_SetFieldBinary(cogr_feature, i, len(value), - string_c) - elif isinstance(value, str): - value_bytes = strencode(value, encoding) - string_c = value_bytes - OGR_F_SetFieldString(cogr_feature, i, string_c) - elif value is None: + if value is None: OGR_F_SetFieldNull(cogr_feature, i) else: - raise ValueError("Invalid field type %s" % type(value)) + schema_type = session._schema_normalized_field_types[key] + fieldkey = (*FIELD_TYPES_MAP2[NAMED_FIELD_TYPES[schema_type]], type(value).__name__) + if fieldkey in self.property_setter_cache: + setter = self.property_setter_cache[fieldkey] + else: + try: + setter = self.OGRPropertySetter[fieldkey](driver=self.driver) + self.property_setter_cache[fieldkey] = setter + except KeyError: + log.warning( + "Skipping field %s: invalid type %s", + key, + fieldkey + ) + continue + + # Special case: serialize dicts to assist OGR. + if isinstance(value, dict): + value = json.dumps(value) + + log.debug("Setting feature property: fieldkey=%r, setter=%r, i=%r, value=%r", fieldkey, setter, i, value) + setter.set(cogr_feature, i, value, {"encoding": encoding}) + return cogr_feature @@ -539,8 +809,6 @@ def featureRT(feat, collection): return result -# Collection-related extension classes and functions - cdef class Session: cdef void *cogr_ds @@ -550,6 +818,21 @@ cdef class Session: cdef object collection cdef bint cursor_interrupted + OGRFieldGetter = { + (OFTInteger, OFSTNone): IntegerField, + (OFTInteger, OFSTBoolean): BooleanField, + (OFTInteger, OFSTInt16): Int16Field, + (OFTInteger64, OFSTNone): Integer64Field, + (OFTReal, OFSTNone): RealField, + (OFTString, OFSTNone): StringField, + (OFTDate, OFSTNone): DateField, + (OFTTime, OFSTNone): TimeField, + (OFTDateTime, OFSTNone): DateTimeField, + (OFTBinary, OFSTNone): BinaryField, + (OFTStringList, OFSTNone): StringListField, + (OFTString, OFSTJSON): JSONField, + } + def __init__(self): self.cogr_ds = NULL self.cogr_layer = NULL @@ -725,9 +1008,11 @@ cdef class Session: """ cdef int i cdef int num_fields - cdef void *cogr_featuredefn = NULL - cdef void *cogr_fielddefn = NULL + cdef OGRFeatureDefnH featuredefn = NULL + cdef OGRFieldDefnH fielddefn = NULL cdef const char *key_c + cdef AbstractField getter + props = {} if self.cogr_layer == NULL: @@ -738,22 +1023,21 @@ cdef class Session: else: ignore_fields = set() - cogr_featuredefn = OGR_L_GetLayerDefn(self.cogr_layer) + featuredefn = OGR_L_GetLayerDefn(self.cogr_layer) - if cogr_featuredefn == NULL: + if featuredefn == NULL: raise ValueError("Null feature definition") encoding = self._get_internal_encoding() - - num_fields = OGR_FD_GetFieldCount(cogr_featuredefn) + num_fields = OGR_FD_GetFieldCount(featuredefn) for i from 0 <= i < num_fields: - cogr_fielddefn = OGR_FD_GetFieldDefn(cogr_featuredefn, i) + fielddefn = OGR_FD_GetFieldDefn(featuredefn, i) - if cogr_fielddefn == NULL: + if fielddefn == NULL: raise ValueError(f"NULL field definition at index {i}") - key_c = OGR_Fld_GetNameRef(cogr_fielddefn) + key_c = OGR_Fld_GetNameRef(fielddefn) if key_c == NULL: raise ValueError(f"NULL field name reference at index {i}") @@ -777,62 +1061,25 @@ cdef class Session: ) continue - fieldtypename = FIELD_TYPES[OGR_Fld_GetType(cogr_fielddefn)] - fieldsubtype = OGR_Fld_GetSubType(cogr_fielddefn) + fieldtype = OGR_Fld_GetType(fielddefn) + fieldsubtype = OGR_Fld_GetSubType(fielddefn) + fieldkey = (fieldtype, fieldsubtype) - if not fieldtypename: + try: + getter = self.OGRFieldGetter[fieldkey](driver=self.collection.driver) + props[key] = getter.name(fielddefn) + except KeyError: log.warning( "Skipping field %s: invalid type %s", key, - OGR_Fld_GetType(cogr_fielddefn)) + fieldkey + ) continue - if fieldtypename == "float": - width = OGR_Fld_GetWidth(cogr_fielddefn) - precision = OGR_Fld_GetPrecision(cogr_fielddefn) - fmt = "" - if width: - fmt = f":{width:d}" - if precision: - fmt += f".{precision:d}" - val = f"float{fmt}" - - elif fieldtypename == "int32": - if fieldsubtype == OFSTBoolean: - val = "bool" - elif fieldsubtype == OFSTInt16: - val = "int16" - else: - fmt = "" - width = OGR_Fld_GetWidth(cogr_fielddefn) - if width: - fmt = f":{width:d}" - val = f"int32{fmt}" - - elif fieldtypename == "int64": - fmt = "" - width = OGR_Fld_GetWidth(cogr_fielddefn) - if width: - fmt = f":{width:d}" - val = f"int{fmt}" - - elif fieldtypename == "str": - fmt = "" - width = OGR_Fld_GetWidth(cogr_fielddefn) - if width: - fmt = f":{width:d}" - val = f"str{fmt}" - - else: - val = fieldtypename - - # Store the field name and description value. - props[key] = val - ret = {"properties": props} if not self.collection.ignore_geometry: - code = normalize_geometry_type_code(OGR_FD_GetGeomType(cogr_featuredefn)) + code = normalize_geometry_type_code(OGR_FD_GetGeomType(featuredefn)) ret["geometry"] = GEOMETRY_TYPES[code] return ret @@ -932,7 +1179,7 @@ cdef class Session: fid = int(fid) cogr_feature = OGR_L_GetFeature(self.cogr_layer, fid) if cogr_feature != NULL: - feature = FeatureBuilder().build( + feature = FeatureBuilder(driver=self.collection.driver).build( cogr_feature, encoding=self._get_internal_encoding(), bbox=False, @@ -966,7 +1213,7 @@ cdef class Session: cogr_feature = OGR_L_GetFeature(self.cogr_layer, index) if cogr_feature == NULL: return None - feature = FeatureBuilder().build( + feature = FeatureBuilder(driver=self.collection.driver).build( cogr_feature, encoding=self._get_internal_encoding(), bbox=False, @@ -1271,11 +1518,10 @@ cdef class WritingSession(Session): # Next, make a layer definition from the given schema properties, # which are a dict built-in since Fiona 2.0 - encoding = self._get_internal_encoding() - # Test if default fields are included in provided schema schema_fields = collection.schema['properties'] default_fields = self.get_schema()['properties'] + for key, value in default_fields.items(): if key in schema_fields and not schema_fields[key] == value: raise SchemaError( @@ -1283,29 +1529,17 @@ cdef class WritingSession(Session): f"for driver '{self.collection.driver}'" ) - new_fields = {key: value for key, value in schema_fields.items() - if key not in default_fields} + new_fields = {k: v for k, v in schema_fields.items() if k not in default_fields} before_fields = default_fields.copy() before_fields.update(new_fields) - for key, value in new_fields.items(): - field_subtype = OFSTNone - - # Convert 'long' to 'int'. See - # https://github.com/Toblerity/Fiona/issues/101. - if value in ('int', 'long'): - value = 'int64' - - elif value == 'bool': - value = 'int32' - field_subtype = OFSTBoolean + encoding = self._get_internal_encoding() - elif value == 'int16': - value = 'int32' - field_subtype = OFSTInt16 + for key, value in new_fields.items(): # Is there a field width/precision? width = precision = None + if ':' in value: value, fmt = value.split(':') @@ -1314,24 +1548,30 @@ cdef class WritingSession(Session): else: width = int(fmt) + # Type inference based on field width is something + # we should reconsider down the road. if value == 'int': if width == 0 or width >= 10: value = 'int64' else: value = 'int32' - field_type = FIELD_TYPES.index(value) + value = normalize_field_type(value) + ftype = NAMED_FIELD_TYPES[value] + ogrfieldtype, ogrfieldsubtype = FIELD_TYPES_MAP2[ftype] try: - key_bytes = key.encode(encoding) - cogr_fielddefn = exc_wrap_pointer(OGR_Fld_Create(key_bytes, field_type)) + key_b = key.encode(encoding) + cogr_fielddefn = exc_wrap_pointer( + OGR_Fld_Create(key_b, ogrfieldtype) + ) + if width: OGR_Fld_SetWidth(cogr_fielddefn, width) if precision: OGR_Fld_SetPrecision(cogr_fielddefn, precision) - if field_subtype != OFSTNone: - # subtypes are new in GDAL 2.x, ignored in 1.x - OGR_Fld_SetSubType(cogr_fielddefn, field_subtype) + if ogrfieldsubtype != OFSTNone: + OGR_Fld_SetSubType(cogr_fielddefn, ogrfieldsubtype) exc_wrap_int(OGR_L_CreateField(self.cogr_layer, cogr_fielddefn, 1)) @@ -1347,8 +1587,7 @@ cdef class WritingSession(Session): # Mapping of the Python collection schema to the munged # OGR schema. after_fields = self.get_schema()['properties'] - self._schema_mapping = dict(zip(before_fields.keys(), - after_fields.keys())) + self._schema_mapping = dict(zip(before_fields.keys(), after_fields.keys())) # Mapping of the Python collection schema to OGR field indices. # We assume that get_schema()['properties'].keys() is in the exact OGR field order @@ -1358,7 +1597,6 @@ cdef class WritingSession(Session): # Mapping of the Python collection schema to normalized field types self._schema_normalized_field_types = {k: normalize_field_type(v) for (k, v) in self.collection.schema['properties'].items()} - log.debug("Writing started") def writerecs(self, records, collection): @@ -1376,14 +1614,15 @@ cdef class WritingSession(Session): None """ - cdef void *cogr_driver - cdef void *cogr_feature + cdef OGRSFDriverH cogr_driver + cdef OGRFeatureH cogr_feature cdef int features_in_transaction = 0 - cdef void *cogr_layer = self.cogr_layer + cdef OGRLayerH cogr_layer = self.cogr_layer if cogr_layer == NULL: raise ValueError("Null layer") + cdef OGRFeatureBuilder feat_builder = OGRFeatureBuilder(driver=collection.driver) valid_geom_types = collection._valid_geom_types def validate_geometry_type(record): @@ -1419,7 +1658,7 @@ cdef class WritingSession(Session): record.geometry.type, collection.schema['geometry'] )) - cogr_feature = OGRFeatureBuilder().build(record, collection) + cogr_feature = feat_builder.build(record, collection) result = OGR_L_CreateFeature(cogr_layer, cogr_feature) if result != OGRERR_NONE: @@ -1543,7 +1782,6 @@ cdef class Iterator: """Provides iterated access to feature data. """ - # Reference to its Collection cdef collection cdef encoding cdef int next_index @@ -1554,6 +1792,7 @@ cdef class Iterator: cdef fastcount cdef ftcount cdef stepsign + cdef FeatureBuilder feat_builder def __cinit__(self, collection, start=None, stop=None, step=None, bbox=None, mask=None, where=None): @@ -1665,6 +1904,8 @@ cdef class Iterator: exc_wrap_int(OGR_L_SetNextByIndex(session.cogr_layer, self.next_index)) session.cursor_interrupted = False + self.feat_builder = FeatureBuilder(driver=collection.driver) + def __iter__(self): return self @@ -1747,7 +1988,7 @@ cdef class Iterator: raise StopIteration try: - return FeatureBuilder().build( + return self.feat_builder.build( cogr_feature, encoding=self.collection.session._get_internal_encoding(), bbox=False, @@ -1779,7 +2020,7 @@ cdef class ItemsIterator(Iterator): try: fid = OGR_F_GetFID(cogr_feature) - feature = FeatureBuilder().build( + feature = self.feat_builder.build( cogr_feature, encoding=self.collection.session._get_internal_encoding(), bbox=False, diff --git a/fiona/rfc3339.py b/fiona/rfc3339.py index d28ac551..d736745d 100644 --- a/fiona/rfc3339.py +++ b/fiona/rfc3339.py @@ -9,22 +9,6 @@ log = logging.getLogger("Fiona") - -# Fiona's 'date', 'time', and 'datetime' types are sub types of 'str'. - - -class FionaDateType(str): - """Dates without time.""" - - -class FionaTimeType(str): - """Times without dates.""" - - -class FionaDateTimeType(str): - """Dates and times.""" - - pattern_date = re.compile(r"(\d\d\d\d)(-)?(\d\d)(-)?(\d\d)") pattern_time = re.compile( r"(\d\d)(:)?(\d\d)(:)?(\d\d)?(\.\d+)?(Z|([+-])?(\d\d)?(:)?(\d\d))?") diff --git a/fiona/schema.pyx b/fiona/schema.pyx index 3ea0f3f9..e0c8d354 100644 --- a/fiona/schema.pyx +++ b/fiona/schema.pyx @@ -1,27 +1,127 @@ """Fiona schema module.""" -from typing import List +include "gdal.pxi" +import itertools from typing import List from fiona.errors import SchemaError -from fiona.rfc3339 import FionaDateType, FionaDateTimeType, FionaTimeType -cdef extern from "gdal.h": - char * GDALVersionInfo (char *pszRequest) +cdef class FionaIntegerType: + names = ["int32"] + type = int + + +cdef class FionaInt16Type: + names = ["int16"] + type = int + + +cdef class FionaBooleanType: + names = ["bool"] + type = bool + + +cdef class FionaInteger64Type: + names = ["int", "int64"] + type = int + + +cdef class FionaRealType: + names = ["float", "float64"] + type = float + + +cdef class FionaStringType: + names = ["str"] + type = str + + +cdef class FionaBinaryType: + names = ["bytes"] + type = bytes + + +cdef class FionaStringListType: + names = ["List[str]", "list[str]"] + type = List[str] + + +cdef class FionaJSONType: + names = ["json"] + type = str + + +cdef class FionaDateType: + """Dates without time.""" + names = ["date"] + type = str -def _get_gdal_version_num(): - """Return current internal version number of gdal""" - return int(GDALVersionInfo("VERSION_NUM")) +cdef class FionaTimeType: + """Times without dates.""" + names = ["time"] + type = str -GDAL_VERSION_NUM = _get_gdal_version_num() +cdef class FionaDateTimeType: + """Dates and times.""" + names = ["datetime"] + type = str + + +FIELD_TYPES_MAP2_REV = { + (OFTInteger, OFSTNone): FionaIntegerType, + (OFTInteger, OFSTBoolean): FionaBooleanType, + (OFTInteger, OFSTInt16): FionaInt16Type, + (OFTInteger64, OFSTNone): FionaInteger64Type, + (OFTReal, OFSTNone): FionaRealType, + (OFTString, OFSTNone): FionaStringType, + (OFTDate, OFSTNone): FionaDateType, + (OFTTime, OFSTNone): FionaTimeType, + (OFTDateTime, OFSTNone): FionaDateTimeType, + (OFTBinary, OFSTNone): FionaBinaryType, + (OFTStringList, OFSTNone): FionaStringListType, + (OFTString, OFSTJSON): FionaJSONType, +} +FIELD_TYPES_MAP2 = {v: k for k, v in FIELD_TYPES_MAP2_REV.items()} +FIELD_TYPES_NAMES = list(itertools.chain.from_iterable((k.names for k in FIELD_TYPES_MAP2))) +NAMED_FIELD_TYPES = {n: k for k in FIELD_TYPES_MAP2 for n in k.names} + + +def normalize_field_type(ftype): + """Normalize free form field types to an element of FIELD_TYPES + + Parameters + ---------- + ftype : str + A type:width format like 'int:9' or 'str:255' + + Returns + ------- + str + An element from FIELD_TYPES + """ + if ftype in FIELD_TYPES_NAMES: + return ftype + elif ftype.startswith('int'): + width = int((ftype.split(':')[1:] or ['0'])[0]) + if width == 0 or width >= 10: + return 'int64' + else: + return 'int32' + elif ftype.startswith('str'): + return 'str' + elif ftype.startswith('float'): + return 'float' + else: + raise SchemaError(f"Unknown field type: {ftype}") + -# Mapping of OGR integer field types to Fiona field type names. -# Lists are currently unsupported in this version, but might be done as -# arrays in a future version. +# Fiona field type names indexed by their major OGR integer field type. +# This data is deprecated, no longer used by the project and is left +# only for other projects that import it. FIELD_TYPES = [ 'int32', # OFTInteger, Simple 32bit integer None, # OFTIntegerList, List of 32bit integers @@ -40,6 +140,8 @@ FIELD_TYPES = [ ] # Mapping of Fiona field type names to Python types. +# This data is deprecated, no longer used by the project and is left +# only for other projects that import it. FIELD_TYPES_MAP = { 'int32': int, 'float': float, @@ -52,39 +154,7 @@ FIELD_TYPES_MAP = { 'int': int, 'List[str]': List[str], } - FIELD_TYPES_MAP_REV = dict([(v, k) for k, v in FIELD_TYPES_MAP.items()]) FIELD_TYPES_MAP_REV[int] = 'int' -def normalize_field_type(ftype): - """Normalize free form field types to an element of FIELD_TYPES - - Parameters - ---------- - ftype : str - A type:width format like 'int:9' or 'str:255' - - Returns - ------- - str - An element from FIELD_TYPES - """ - if ftype in FIELD_TYPES: - return ftype - elif ftype == 'bool': - return 'bool' - elif ftype == "int16": - return 'int32' - elif ftype.startswith('int'): - width = int((ftype.split(':')[1:] or ['0'])[0]) - if GDAL_VERSION_NUM >= 2000000 and (width == 0 or width >= 10): - return 'int64' - else: - return 'int32' - elif ftype.startswith('str'): - return 'str' - elif ftype.startswith('float'): - return 'float' - else: - raise SchemaError(f"Unknown field type: {ftype}") diff --git a/tests/conftest.py b/tests/conftest.py index 18584f49..1c5dfa8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -202,7 +202,6 @@ def path_testopenfilegdb_zip(data_dir): return os.path.join(data_dir, "testopenfilegdb.gdb.zip") - @pytest.fixture(scope="session") def bytes_testopenfilegdb_zip(path_testopenfilegdb_zip): """.gdb.zip bytes.""" diff --git a/tests/test_props.py b/tests/test_props.py index e3e1fdba..21a14275 100644 --- a/tests/test_props.py +++ b/tests/test_props.py @@ -2,10 +2,13 @@ import os.path import tempfile +import pytest + import fiona from fiona import prop_type, prop_width from fiona.model import Feature -from fiona.rfc3339 import FionaDateType + +from .conftest import gdal_version def test_width_str(): @@ -24,9 +27,10 @@ def test_types(): assert prop_type("str") == str assert isinstance(0, prop_type("int")) assert isinstance(0.0, prop_type("float")) - assert prop_type("date") == FionaDateType + assert prop_type("date") == str +@pytest.mark.xfail(not gdal_version.at_least("3.5"), reason="Requires at least GDAL 3.5.0") def test_read_json_object_properties(): """JSON object properties are properly serialized""" data = """ @@ -87,6 +91,7 @@ def test_read_json_object_properties(): assert props["tricky"] == "{gotcha" +@pytest.mark.xfail(not gdal_version.at_least("3.5"), reason="Requires at least GDAL 3.5.0") def test_write_json_object_properties(): """Python object properties are properly serialized""" data = """ diff --git a/tests/test_schema.py b/tests/test_schema.py index db5b9a36..03338174 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -8,7 +8,7 @@ from fiona.env import GDALVersion from fiona.errors import SchemaError, UnsupportedGeometryTypeError, DriverSupportError from fiona.model import Feature -from fiona.schema import FIELD_TYPES, normalize_field_type +from fiona.schema import NAMED_FIELD_TYPES, normalize_field_type from .conftest import get_temp_filename from .conftest import requires_only_gdal1, requires_gdal2 @@ -200,15 +200,18 @@ def test_normalize_float(): assert normalize_field_type("float:25.8") == "float" +def test_normalize_(): + assert normalize_field_type("float:25.8") == "float" + + def generate_field_types(): """ Produce a unique set of field types in a consistent order. This ensures that tests are able to run in parallel. """ - types = set(FIELD_TYPES) - types.remove(None) - return list(sorted(types)) + [None] + types = set(NAMED_FIELD_TYPES.keys()) + return list(sorted(types)) @pytest.mark.parametrize("x", generate_field_types())