From c13e305f4e5687c06d840b991cc061fd3cb643ef Mon Sep 17 00:00:00 2001 From: Jeff VanOss Date: Mon, 1 Oct 2018 12:23:33 -0400 Subject: [PATCH 1/4] populate Py_buffer strides field in ctypes arrays --- Lib/ctypes/test/test_pep3118.py | 100 +++++++++------- .../2017-12-08-19-58-16.bpo-12851.5H37Gx.rst | 2 + Modules/_ctypes/_ctypes.c | 38 ++++-- Modules/_ctypes/_ctypes_test.c | 109 ++++++++++++++++++ Modules/_ctypes/ctypes.h | 4 +- Modules/_ctypes/stgdict.c | 9 +- 6 files changed, 205 insertions(+), 57 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-12-08-19-58-16.bpo-12851.5H37Gx.rst diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py index 81e8ca7638fdeb..45f9af4b843456 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/ctypes/test/test_pep3118.py @@ -1,6 +1,7 @@ import unittest from ctypes import * import re, sys +from _ctypes_test import buffer_info if sys.byteorder == "little": THIS_ENDIAN = "<" @@ -20,7 +21,7 @@ def normalize(format): class Test(unittest.TestCase): def test_native_types(self): - for tp, fmt, shape, itemtp in native_types: + for tp, fmt, shape, stride, itemtp in native_types: ob = tp() v = memoryview(ob) try: @@ -31,10 +32,7 @@ def test_native_types(self): self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob)) self.assertEqual(v.itemsize, sizeof(itemtp)) self.assertEqual(v.shape, shape) - # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides - # if requested. memoryview currently reconstructs missing - # stride information, so this assert will fail. - # self.assertEqual(v.strides, ()) + self.assertEqual(v.strides, stride) # they are always read/write self.assertFalse(v.readonly) @@ -49,6 +47,23 @@ def test_native_types(self): print(tp) raise + def test_native_types_shape_strides(self): + # check that ctypes (not memoryview) correctly fills out shape and strides in buffer protocol + for tp, fmt, shape, stride, itemtp in native_types: + ob = tp() + v = buffer_info(ob) + try: + if v['ndim'] == 0: + self.assertEqual(v['shape'], None) + self.assertEqual(v['strides'], None) + else: + self.assertEqual(v['shape'], shape) + self.assertEqual(v['strides'], stride) + except: + # so that we can see the failing type + print(tp) + raise + def test_endian_types(self): for tp, fmt, shape, itemtp in endian_types: ob = tp() @@ -61,8 +76,7 @@ def test_endian_types(self): self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob)) self.assertEqual(v.itemsize, sizeof(itemtp)) self.assertEqual(v.shape, shape) - # XXX Issue #12851 - # self.assertEqual(v.strides, ()) + self.assertEqual(v.strides, ()) # they are always read/write self.assertFalse(v.readonly) @@ -141,73 +155,73 @@ class Complete(Structure): native_types = [ - # type format shape calc itemsize + # type format shape stride calc itemsize ## simple types - (c_char, "format == NULL) goto error; stgdict->ndim = itemdict->ndim + 1; - stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stgdict->ndim); + stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stgdict->ndim * 2); if (stgdict->shape == NULL) { PyErr_NoMemory(); goto error; @@ -1505,6 +1505,14 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) memmove(&stgdict->shape[1], itemdict->shape, sizeof(Py_ssize_t) * (stgdict->ndim - 1)); } + stgdict->strides = stgdict->shape + stgdict->ndim; + if (stgdict->ndim > 1) { + memmove(&stgdict->strides[1], itemdict->strides, + sizeof(Py_ssize_t) * (stgdict->ndim - 1)); + stgdict->strides[0] = stgdict->strides[1] * stgdict->shape[1]; + } else { + stgdict->strides[0] = itemdict->size; + } itemsize = itemdict->size; if (length * itemsize < 0) { @@ -2681,26 +2689,34 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; StgDictObject *dict = PyObject_stgdict(myself); - Py_ssize_t i; if (view == NULL) return 0; + if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) { + view->obj = NULL; + PyErr_Format(PyExc_TypeError, "Fortran contiguous buffer is not supported"); + return -1; + } + view->buf = self->b_ptr; view->obj = myself; Py_INCREF(myself); view->len = self->b_size; view->readonly = 0; - /* use default format character if not set */ - view->format = dict->format ? dict->format : "B"; + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + /* use default format character if not set */ + view->format = dict->format ? dict->format : "B"; + } else { + view->format = NULL; + } view->ndim = dict->ndim; - view->shape = dict->shape; - view->itemsize = self->b_size; - if (view->itemsize) { - for (i = 0; i < view->ndim; ++i) { - view->itemsize /= dict->shape[i]; - } + view->shape = ((flags & PyBUF_ND) == PyBUF_ND) ? dict->shape : NULL; + if (dict->strides) { + view->itemsize = dict->strides[dict->ndim - 1]; + } else { + view->itemsize = self->b_size; } - view->strides = NULL; + view->strides = ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) ? dict->strides : NULL; view->suboffsets = NULL; view->internal = NULL; return 0; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index f8420580ffa811..be4291f30f191e 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -475,12 +475,121 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name) return 999; } +PyObject *py_getBufferInfo(PyObject *self, PyObject *obj) +{ + int flags = PyBUF_FULL; + int ii; + Py_buffer view; + PyObject *v; + PyObject *x; + PyObject *d; + int buffer_err; + + d = PyDict_New(); + if (NULL == d) Py_RETURN_NONE; + buffer_err = PyObject_GetBuffer(obj, &view, flags); + v = PyLong_FromLong(buffer_err); + if (NULL != v){ + PyDict_SetItemString(d, "err", v); + Py_DECREF(v); + } + + if (0 == buffer_err) { + v = PyLong_FromSsize_t(view.len); + if (NULL != v){ + PyDict_SetItemString(d, "len", v); + Py_DECREF(v); + } + v = PyBool_FromLong(view.readonly); + if (NULL != v){ + PyDict_SetItemString(d, "readonly", v); + Py_DECREF(v); + } + v = PyLong_FromSsize_t(view.itemsize); + if (NULL != v){ + PyDict_SetItemString(d, "itemsize", v); + Py_DECREF(v); + } + if(NULL == view.format) { + v = PyUnicode_New(0, 0); + } else { + v = PyUnicode_FromString(view.format); + } + if (NULL != v){ + PyDict_SetItemString(d, "format", v); + Py_DECREF(v); + } + v = PyLong_FromLong(view.ndim); + if (NULL != v){ + PyDict_SetItemString(d, "ndim", v); + Py_DECREF(v); + } + if (NULL == view.shape) { + PyDict_SetItemString(d, "shape", Py_None); + } else { + v = PyTuple_New(view.ndim); + if (NULL != v){ + for (ii=0; iiformat = NULL; self->ndim = 0; self->shape = NULL; + self->strides = NULL; return 0; } @@ -75,6 +76,7 @@ PyCStgDict_clone(StgDictObject *dst, StgDictObject *src) dst->format = NULL; PyMem_Free(dst->shape); dst->shape = NULL; + dst->strides = NULL; dst->ffi_type_pointer.elements = NULL; d = (char *)dst; @@ -98,13 +100,18 @@ PyCStgDict_clone(StgDictObject *dst, StgDictObject *src) strcpy(dst->format, src->format); } if (src->shape) { - dst->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src->ndim); + dst->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src->ndim * 2); if (dst->shape == NULL) { PyErr_NoMemory(); return -1; } memcpy(dst->shape, src->shape, sizeof(Py_ssize_t) * src->ndim); + if(src->strides) { + dst->strides = dst->shape + src->ndim; + memcpy(dst->strides, src->strides, + sizeof(Py_ssize_t) * src->ndim); + } } if (src->ffi_type_pointer.elements == NULL) From f6be10ae3c91b2d78ff66bdaab6171f0deca39e1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 11 Aug 2026 20:25:28 +0300 Subject: [PATCH 2/4] Do not report ndim > 1 without the shape PyBUF_SIMPLE requests crashed on a multidimensional array: ndim > 1 implies shape != NULL (see PyBuffer_IsContiguous()). The buffer is C contiguous, so it can be exposed as flat. Also, only a multidimensional array is not Fortran contiguous, and BufferError is more appropriate than TypeError. Clear the strides when the shape which they point into is freed. --- Modules/_ctypes/_ctypes.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index a68977b2ca8b16..744e82e7251994 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -502,8 +502,10 @@ ctype_free_stginfo_members(StgInfo *info) info->ffi_type_pointer.elements = NULL; PyMem_Free(info->format); info->format = NULL; + /* The strides point into the shape allocation. */ PyMem_Free(info->shape); info->shape = NULL; + info->strides = NULL; ctype_clear_stginfo(info); } @@ -3134,9 +3136,9 @@ PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) } assert(item_info); - if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) { - view->obj = NULL; - PyErr_Format(PyExc_TypeError, "Fortran contiguous buffer is not supported"); + if (info->ndim > 1 && (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) { + PyErr_SetString(PyExc_BufferError, + "ctypes array is not Fortran contiguous"); return -1; } @@ -3151,10 +3153,21 @@ PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) else { view->format = NULL; } - view->ndim = info->ndim; - view->shape = ((flags & PyBUF_ND) == PyBUF_ND) ? info->shape : NULL; + if ((flags & PyBUF_ND) == PyBUF_ND) { + view->ndim = info->ndim; + view->shape = info->shape; + } + else { + /* The buffer is C contiguous, so it can be exposed as flat. + Keep ndim <= 1: ndim > 1 implies shape != NULL, see + PyBuffer_IsContiguous(). */ + view->ndim = info->ndim ? 1 : 0; + view->shape = NULL; + } view->itemsize = item_info->size; - view->strides = ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) ? info->strides : NULL; + /* PyBUF_STRIDES implies PyBUF_ND. */ + view->strides = ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) + ? info->strides : NULL; view->suboffsets = NULL; view->internal = NULL; return 0; From 03b0de1ad101aedeaa5c9108c62bb4d07141e59b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 11 Aug 2026 20:26:37 +0300 Subject: [PATCH 3/4] Use _testbuffer instead of a new helper in _ctypes_test _testbuffer.ndarray already exposes the raw Py_buffer, and it can also request particular flags, which the buffer_info() helper could not. --- Lib/test/test_ctypes/test_pep3118.py | 48 +++++++++--- Modules/_ctypes/_ctypes_test.c | 109 --------------------------- 2 files changed, 38 insertions(+), 119 deletions(-) diff --git a/Lib/test/test_ctypes/test_pep3118.py b/Lib/test/test_ctypes/test_pep3118.py index 5fb818fe4de245..a9f8a5a8af37c9 100644 --- a/Lib/test/test_ctypes/test_pep3118.py +++ b/Lib/test/test_ctypes/test_pep3118.py @@ -1,7 +1,7 @@ import re import sys import unittest -from _ctypes_test import buffer_info +from test.support import import_helper from ctypes import (CFUNCTYPE, POINTER, sizeof, Union, Structure, LittleEndianStructure, BigEndianStructure, c_char, c_byte, c_ubyte, @@ -9,6 +9,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_uint64, c_bool, c_float, c_double, c_longdouble, py_object) +_testbuffer = import_helper.import_module('_testbuffer') + if sys.byteorder == "little": THIS_ENDIAN = "<" @@ -50,17 +52,43 @@ def test_native_types(self): self.assertEqual(n * v.itemsize, len(v.tobytes())) def test_native_types_shape_strides(self): - # check that ctypes (not memoryview) correctly fills out shape and - # strides in the buffer protocol + # memoryview fills in the shape and the strides which the exporter + # does not provide, and always requests all of them, so check what + # ctypes exports itself. for tp, fmt, shape, stride, itemtp in native_types: with self.subTest(tp=tp): - v = buffer_info(tp()) - if v['ndim'] == 0: - self.assertIsNone(v['shape']) - self.assertIsNone(v['strides']) - else: - self.assertEqual(v['shape'], shape) - self.assertEqual(v['strides'], stride) + v = _testbuffer.ndarray(tp(), getbuf=_testbuffer.PyBUF_FULL_RO) + self.assertEqual(v.shape, shape) + self.assertEqual(v.strides, stride) + + def test_flags(self): + ob = (c_int * 3 * 2)() + + v = _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_SIMPLE) + # ndim > 1 implies shape != NULL, so a flat buffer is exported. + self.assertEqual(v.ndim, 1) + self.assertEqual(v.shape, ()) + self.assertEqual(v.strides, ()) + + v = _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_ND) + self.assertEqual(v.shape, (2, 3)) + self.assertEqual(v.strides, ()) + + v = _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_STRIDES) + self.assertEqual(v.shape, (2, 3)) + self.assertEqual(v.strides, (12, 4)) + + def test_fortran_contiguous(self): + # A multidimensional array is C contiguous, but not Fortran + # contiguous. A one-dimensional array is contiguous in both orders. + ob = (c_int * 3 * 2)() + with self.assertRaises(BufferError): + _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_F_CONTIGUOUS) + _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_C_CONTIGUOUS) + + ob = (c_int * 3)() + _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_F_CONTIGUOUS) + _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_C_CONTIGUOUS) def test_endian_types(self): for tp, fmt, shape, stride, itemtp in endian_types: diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 1efe6659a6c7e2..991ff0d675c2f1 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -852,120 +852,11 @@ EXPORT(TestReg) get_last_tfrsuv_arg(void) return last_tfrsuv_arg; } -PyObject *py_getBufferInfo(PyObject *self, PyObject *obj) -{ - int flags = PyBUF_FULL; - int ii; - Py_buffer view; - PyObject *v; - PyObject *x; - PyObject *d; - int buffer_err; - - d = PyDict_New(); - if (NULL == d) Py_RETURN_NONE; - buffer_err = PyObject_GetBuffer(obj, &view, flags); - v = PyLong_FromLong(buffer_err); - if (NULL != v){ - PyDict_SetItemString(d, "err", v); - Py_DECREF(v); - } - - if (0 == buffer_err) { - v = PyLong_FromSsize_t(view.len); - if (NULL != v){ - PyDict_SetItemString(d, "len", v); - Py_DECREF(v); - } - v = PyBool_FromLong(view.readonly); - if (NULL != v){ - PyDict_SetItemString(d, "readonly", v); - Py_DECREF(v); - } - v = PyLong_FromSsize_t(view.itemsize); - if (NULL != v){ - PyDict_SetItemString(d, "itemsize", v); - Py_DECREF(v); - } - if(NULL == view.format) { - v = PyUnicode_FromString(""); - } else { - v = PyUnicode_FromString(view.format); - } - if (NULL != v){ - PyDict_SetItemString(d, "format", v); - Py_DECREF(v); - } - v = PyLong_FromLong(view.ndim); - if (NULL != v){ - PyDict_SetItemString(d, "ndim", v); - Py_DECREF(v); - } - if (NULL == view.shape) { - PyDict_SetItemString(d, "shape", Py_None); - } else { - v = PyTuple_New(view.ndim); - if (NULL != v){ - for (ii=0; ii Date: Tue, 11 Aug 2026 20:51:54 +0300 Subject: [PATCH 4/4] Rename the NEWS entry to the GitHub issue and describe the change --- .../next/Library/2017-12-08-19-58-16.bpo-12851.5H37Gx.rst | 2 -- .../Library/2017-12-08-19-58-16.gh-issue-57060.5H37Gx.rst | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2017-12-08-19-58-16.bpo-12851.5H37Gx.rst create mode 100644 Misc/NEWS.d/next/Library/2017-12-08-19-58-16.gh-issue-57060.5H37Gx.rst diff --git a/Misc/NEWS.d/next/Library/2017-12-08-19-58-16.bpo-12851.5H37Gx.rst b/Misc/NEWS.d/next/Library/2017-12-08-19-58-16.bpo-12851.5H37Gx.rst deleted file mode 100644 index 4ee0d5bb3e8967..00000000000000 --- a/Misc/NEWS.d/next/Library/2017-12-08-19-58-16.bpo-12851.5H37Gx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed ctypes Arrays Py_buffer implimentation to provide strides if -requested. diff --git a/Misc/NEWS.d/next/Library/2017-12-08-19-58-16.gh-issue-57060.5H37Gx.rst b/Misc/NEWS.d/next/Library/2017-12-08-19-58-16.gh-issue-57060.5H37Gx.rst new file mode 100644 index 00000000000000..33b20401f256d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-08-19-58-16.gh-issue-57060.5H37Gx.rst @@ -0,0 +1,4 @@ +The buffer exported by :mod:`ctypes` arrays now provides strides if they are +requested, as required by :pep:`3118`. The format and the shape are now +provided only if they are requested, and requesting a Fortran contiguous +buffer for a multidimensional array now fails.