Skip to content

Commit

Permalink
python-driver: support WKB geometries
Browse files Browse the repository at this point in the history
* Add support for passing ISO WKB geometries from Python OGR drivers
  by using a bytes-like object (bytes, bytesarray, memoryview).
* Expand the dummy python driver in tests to take options, and use them
  to test the geometry output format.
* Update the documentation and examples; tidy up some formatting.
  • Loading branch information
rcoup committed Sep 14, 2023
1 parent 3037540 commit 6db4096
Show file tree
Hide file tree
Showing 7 changed files with 96 additions and 41 deletions.
24 changes: 19 additions & 5 deletions autotest/ogr/data/pydrivers/ogr_DUMMY.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# gdal: DRIVER_SUPPORTED_API_VERSION = [1]
# gdal: DRIVER_DCAP_VECTOR = "YES"
# gdal: DRIVER_DMD_LONGNAME = "my super plugin"
# gdal: DRIVER_DMD_OPENOPTIONLIST = "<OpenOptionList><Option name='GEOMFORMAT' type='string-select' description='Geometry format' default='WKT'><Value>WKT</Value><Value>WKB</Value><Value>WKB/bytearray</Value></Option></OpenOptionList>"

# Optional driver metadata items.
# # gdal: DRIVER_DMD_EXTENSIONS = "ext1 est2"
Expand All @@ -35,7 +36,8 @@ class BaseLayer(object):


class Layer(BaseLayer):
def __init__(self):
def __init__(self, options):
self.options = options

# Reserved attribute names. Either those or the corresponding method
# must be defined
Expand Down Expand Up @@ -167,12 +169,23 @@ def __iter__(self):
"dateField": "2017-04-26",
"datetimeField": "2017-04-26T12:34:56.789Z",
}
POINT_WKB = b"\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x80H@"

geom_format = self.options.get("GEOMFORMAT", "WKT")
if geom_format == "WKT":
geom = "POINT(2 49)"
elif geom_format == "WKB":
geom = POINT_WKB
elif geom_format == "WKB/bytearray":
geom = bytearray(POINT_WKB)
else:
raise ValueError(f"Unknown GEOMFORMAT: {geom_format}")

yield {
"type": "OGRFeature",
"id": i + 1,
"fields": properties,
"geometry_fields": {"geomField": "POINT(2 49)"},
"geometry_fields": {"geomField": geom},
"style": "SYMBOL(a:0)" if i % 2 == 0 else None,
}

Expand All @@ -184,10 +197,11 @@ def __iter__(self):
class Dataset(BaseDataset):

# Optional, but implementations will generally need it
def __init__(self, filename):
def __init__(self, filename, options):
# If the layers member is set, layer_count() and layer() will not be used
self.layers = [Layer()]
self.layers = [Layer(options)]
self.metadata = {"foo": "bar"}
self.options = options

# Optional, called on native object destruction
def close(self):
Expand Down Expand Up @@ -223,4 +237,4 @@ def identify(self, filename, first_bytes, open_flags, open_options={}):
def open(self, filename, first_bytes, open_flags, open_options={}):
if not self.identify(filename, first_bytes, open_flags):
return None
return Dataset(filename)
return Dataset(filename, options=open_options)
10 changes: 7 additions & 3 deletions autotest/ogr/ogr_pythondrivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,11 @@ def setup_and_cleanup():
assert not ogr.GetDriverByName("DUMMY")


def test_pythondrivers_test_dummy():
@pytest.mark.parametrize("geomformat", ["WKT", "WKB", "WKB/bytearray"])
def test_pythondrivers_test_dummy(geomformat):
assert not ogr.Open("UNRELATED:")
ds = ogr.Open("DUMMY:")

ds = gdal.OpenEx("DUMMY:", open_options=["GEOMFORMAT=" + geomformat])
assert ds
assert ds.GetLayerCount() == 1
assert not ds.GetLayer(-1)
Expand Down Expand Up @@ -85,7 +87,9 @@ def test_pythondrivers_test_dummy():
assert f["timeField"] == "12:34:56.789"
assert f["dateField"] == "2017/04/26"
assert f["datetimeField"] == "2017/04/26 12:34:56.789+00"
assert f.GetGeometryRef()
g = f.GetGeometryRef()
assert g is not None
assert g.GetPoint() == (2.0, 49.0, 0.0)
count += 1
assert count == 5
assert lyr.TestCapability(ogr.OLCFastFeatureCount)
Expand Down
53 changes: 28 additions & 25 deletions doc/source/tutorials/vector_python_driver.rst
Original file line number Diff line number Diff line change
Expand Up @@ -81,35 +81,36 @@ constraints:

The following directives must be declared:

* ``# gdal: DRIVER_NAME`` = "some_name": the short name of the driver
* ``# gdal: DRIVER_SUPPORTED_API_VERSION`` = [1]: the API version(s) supported by
* ``# gdal: DRIVER_NAME = "NAME"``: the short name of the driver
* ``# gdal: DRIVER_SUPPORTED_API_VERSION = [1]``: the API version(s) supported by
the driver. Must include 1, which is the only currently supported version in GDAL 3.1
* ``# gdal: DRIVER_DCAP_VECTOR`` = "YES": declares a vector driver
* ``# gdal: DRIVER_DMD_LONGNAME`` = "a longer description of the driver"
* ``# gdal: DRIVER_DCAP_VECTOR = "YES"``: declares a vector driver
* ``# gdal: DRIVER_DMD_LONGNAME = "a longer name of the driver"``

Additional directives:

* ``# gdal: DRIVER_DMD_EXTENSIONS`` = "ext1 ext2": list of extension(s) recognized
* ``# gdal: DRIVER_DMD_EXTENSIONS = "ext1 ext2"``: list of extension(s) recognized
by the driver, without the dot, and separated by space
* ``# gdal: DRIVER_DMD_HELPTOPIC`` = "url_to_hep_page"
* ``# gdal: DRIVER_DMD_OPENOPTIONLIST`` = xml_value where xml_value is an OptionOptionList
specification, like "<OpenOptionList><Option name='OPT1' type='boolean' description='bla' default='NO'/></OpenOptionList>"**
* and all other metadata items found in gdal.h starting with `GDAL_DMD_` (resp. `GDAL_DCAP`) by
creating an item name which starts with `# gdal: DRIVER_` and the value of the
`GDAL_DMD_` (resp. `GDAL_DCAP`) metadata item.
* ``# gdal: DRIVER_DMD_HELPTOPIC = "https://example.com/my_help.html"``: URL to a help
page for the driver
* ``# gdal: DRIVER_DMD_OPENOPTIONLIST = "<OpenOptionList><Option name='OPT1' type='boolean' description='bla' default='NO'/></OpenOptionList>"``
where the XML is an ``OptionOptionList``.
* and all other metadata items found in gdal.h starting with ``GDAL_DMD_`` or ``GDAL_DCAP`` by
creating an item name which starts with ``# gdal: DRIVER_`` and the value of the
``GDAL_DMD_`` or ``GDAL_DCAP`` metadata item.
For example ``#define GDAL_DMD_CONNECTION_PREFIX "DMD_CONNECTION_PREFIX"`` becomes ``# gdal: DRIVER_DMD_CONNECTION_PREFIX``


Example:

.. code-block::
.. code-block::
# gdal: DRIVER_NAME = "DUMMY"
# gdal: DRIVER_SUPPORTED_API_VERSION = [1]
# gdal: DRIVER_DCAP_VECTOR = "YES"
# gdal: DRIVER_DMD_LONGNAME = "my super plugin"
# gdal: DRIVER_DMD_LONGNAME = "my dummy plugin"
# gdal: DRIVER_DMD_EXTENSIONS = "foo bar"
# gdal: DRIVER_DMD_HELPTOPIC = "http://example.com/my_help.html"
# gdal: DRIVER_DMD_HELPTOPIC = "https://example.com/my_help.html"
Driver class
------------
Expand Down Expand Up @@ -295,7 +296,7 @@ The following attributes are required and must defined at __init__ time:

The SRS attached to the geometry field as a string that can be ingested by
:cpp:func:`OGRSpatialReference::SetFromUserInput`, such as a PROJ string,
WKT string, or AUTHORITY:CODE.
WKT string, or ``AUTHORITY:CODE``.

If that attribute is not set, a ``geometry_fields`` method must be defined and
return such a sequence.
Expand Down Expand Up @@ -345,31 +346,33 @@ Feature iterator
The Layer class must implement the iterator interface, so typically with
a ``__iter__`` method.

The iterator must return a dictionary with the feature content.

Two keys allowed in the returned dictionary are:
The resulting iterator must produce dictionaries for each feature's content. The
keys allowed in the returned dictionary are:

.. py:attribute:: id
:noindex:

Strongly recommended. The value must be of type int to be recognized as a FID by GDAL
Strongly recommended. The value must be an integer to be recognized as a FID.

.. py:attribute:: type
:noindex:

Required. The value must be the string "OGRFeature"
Required. The value must be the string ``"OGRFeature"``

.. py:attribute:: fields
:noindex:

Required. The value must be a dictionary whose keys are field names, or None
Required. The value must be either a dictionary whose keys are field names; or None

.. py:attribute:: geometry_fields
:noindex:

Required. the value must be a dictionary whose keys are geometry field names (possibly
the empty string for unnamed geometry columns), or None.
The value of each key must be a geometry encoded as WKT, or None.
the empty string for unnamed geometry columns); or None.

The value of each key must be either a geometry encoded as a WKT string; a geometry
encoded as ISO WKB as a `bytes-like object <https://docs.python.org/3/glossary.html#term-bytes-like-object>`__;
or None.

.. py:attribute:: style
:noindex:
Expand Down Expand Up @@ -469,7 +472,7 @@ attributes, and the ``attribute_filter_changed`` and ``spatial_filter_changed``
method implementations, could have omitted with the same result.

The connection strings recognized by the drivers are
"PASSHTROUGH:connection_string_supported_by_non_python_drivers". Note that
``PASSHTROUGH:connection_string_supported_by_non_python_drivers``. Note that
the prefixing by the driver name is absolutely not a requirement, but something
specific to this particular driver which is a bit artificial (without the prefix,
the connection string would go directly to the native driver). The CityJSON
Expand Down Expand Up @@ -575,7 +578,7 @@ not need it.
g = ogr_f.GetGeomFieldRef(i)
if g:
geom_fields[layer_defn.GetGeomFieldDefn(
i).GetName()] = g.ExportToIsoWkt()
i).GetName()] = g.ExportToIsoWKb()
return {'id': ogr_f.GetFID(),
'type': 'OGRFeature',
'style': ogr_f.GetStyleString(),
Expand Down
2 changes: 1 addition & 1 deletion examples/pydrivers/ogr_PASSTHROUGH.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ def _translate_feature(self, ogr_f):
if g:
geom_fields[
layer_defn.GetGeomFieldDefn(i).GetName()
] = g.ExportToIsoWkt()
] = g.ExportToIsoWkb()
return {
"id": ogr_f.GetFID(),
"type": "OGRFeature",
Expand Down
4 changes: 4 additions & 0 deletions gcore/gdalpython.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ int (*PyTuple_SetItem)(PyObject *, size_t, PyObject *) = nullptr;
void (*PyObject_Print)(PyObject *, FILE *, int) = nullptr;
Py_ssize_t (*PyBytes_Size)(PyObject *) = nullptr;
const char *(*PyBytes_AsString)(PyObject *) = nullptr;
int *(*PyBytes_AsStringAndSize)(PyObject *, char **, size_t *) = nullptr;
PyObject *(*PyBytes_FromObject)(PyObject *) = nullptr;
PyObject *(*PyBytes_FromStringAndSize)(const void *, size_t) = nullptr;
PyObject *(*PyUnicode_FromString)(const char *) = nullptr;
PyObject *(*PyUnicode_AsUTF8String)(PyObject *) = nullptr;
Expand Down Expand Up @@ -726,6 +728,8 @@ static bool LoadPythonAPI()
LOAD(libHandle, PyLong_AsLongLong);
LOAD(libHandle, PyBytes_Size);
LOAD(libHandle, PyBytes_AsString);
LOAD(libHandle, PyBytes_AsStringAndSize);
LOAD(libHandle, PyBytes_FromObject);
LOAD(libHandle, PyBytes_FromStringAndSize);

LOAD(libHandle, PyModule_Create2);
Expand Down
2 changes: 2 additions & 0 deletions gcore/gdalpython.h
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ extern void (*PyObject_Print)(PyObject *, FILE *, int);

extern Py_ssize_t (*PyBytes_Size)(PyObject *);
extern const char *(*PyBytes_AsString)(PyObject *);
extern int *(*PyBytes_AsStringAndSize)(PyObject *, char **, Py_ssize_t *);
extern PyObject *(*PyBytes_FromObject)(PyObject *);
extern PyObject *(*PyBytes_FromStringAndSize)(const void *, size_t);

extern PyObject *(*PyUnicode_FromString)(const char *);
Expand Down
42 changes: 35 additions & 7 deletions gcore/gdalpythondriverloader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,8 @@ OGRFeature *PythonPluginLayer::TranslateToOGRFeature(PyObject *poObj)
PyObject *myLongType = PyObject_Type(myLong);
PyObject *myFloat = PyFloat_FromDouble(1.0);
PyObject *myFloatType = PyObject_Type(myFloat);
PyObject *myStr = PyUnicode_FromString("");
PyObject *myStrType = PyObject_Type(myStr);

auto poFields = PyDict_GetItemString(poObj, "fields");
auto poGeometryFields = PyDict_GetItemString(poObj, "geometry_fields");
Expand Down Expand Up @@ -812,17 +814,41 @@ OGRFeature *PythonPluginLayer::TranslateToOGRFeature(PyObject *poObj)
}
if (value != Py_None)
{
CPLString osValue = GetString(value);
if (ErrOccurredEmitCPLError())
{
break;
}
const int idx = m_poFeatureDefn->GetGeomFieldIndex(osKey);
if (idx >= 0)
{
OGRGeometry *poGeom = nullptr;
OGRGeometryFactory::createFromWkt(osValue.c_str(), nullptr,
&poGeom);
if (PyObject_IsInstance(value, myStrType))
{
// WKT
CPLString osValue = GetString(value);
if (ErrOccurredEmitCPLError())
{
break;
}
OGRGeometryFactory::createFromWkt(osValue.c_str(),
nullptr, &poGeom);
}
else
{
// WKB (from bytes, bytearray, memoryview)
PyObject *poBytes = PyBytes_FromObject(value);
if (ErrOccurredEmitCPLError())
{
break;
}
char *buffer = nullptr;
size_t length = 0;
PyBytes_AsStringAndSize(poBytes, &buffer, &length);
if (ErrOccurredEmitCPLError())
{
break;
}

OGRGeometryFactory::createFromWkb(
buffer, nullptr, &poGeom, length, wkbVariantIso);
}

if (poGeom)
{
const auto poGeomFieldDefn =
Expand Down Expand Up @@ -919,6 +945,8 @@ OGRFeature *PythonPluginLayer::TranslateToOGRFeature(PyObject *poObj)
Py_DecRef(myLong);
Py_DecRef(myFloatType);
Py_DecRef(myFloat);
Py_DecRef(myStr);
Py_DecRef(myStrType);

return poFeature;
}
Expand Down

0 comments on commit 6db4096

Please sign in to comment.