Skip to content

gh-57060: populate Py_buffer strides field for ctypes arrays. #4714

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vanossj wants to merge 6 commits into
base: main
Choose a base branch
from
Open

gh-57060: populate Py_buffer strides field for ctypes arrays. #4714

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c13e305
populate Py_buffer strides field in ctypes arrays
vanossj Oct 1, 2018
41371ae
Merge remote-tracking branch 'upstream/main' into fix-issue-12851
serhiy-storchaka Aug 11, 2026
f6be10a
Do not report ndim > 1 without the shape
serhiy-storchaka Aug 11, 2026
03b0de1
Use _testbuffer instead of a new helper in _ctypes_test
serhiy-storchaka Aug 11, 2026
57c2e6d
Rename the NEWS entry to the GitHub issue and describe the change
serhiy-storchaka Aug 11, 2026
846c8be
Merge branch 'main' into fix-issue-12851
vanossj Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 92 additions & 54 deletions Lib/test/test_ctypes/test_pep3118.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
import re
import sys
import unittest
from test.support import import_helper
from ctypes import (CFUNCTYPE, POINTER, sizeof, Union,
Structure, LittleEndianStructure, BigEndianStructure,
c_char, c_byte, c_ubyte,
c_short, c_ushort, c_int, c_uint,
c_long, c_ulong, c_longlong, c_ulonglong, c_uint64,
c_bool, c_float, c_double, c_longdouble, py_object)

_testbuffer = import_helper.import_module('_testbuffer')


if sys.byteorder == "little":
THIS_ENDIAN = "<"
Expand All @@ -28,7 +31,7 @@ def normalize(format):

class Test(unittest.TestCase):
def test_native_types(self):
for tp, fmt, shape, itemtp in native_types:
for tp, fmt, shape, stride, itemtp in native_types:
ob = tp()
v = memoryview(ob)
self.assertEqual(normalize(v.format), normalize(fmt))
Expand All @@ -38,10 +41,7 @@ def test_native_types(self):
self.assertRaises(TypeError, len, v)
self.assertEqual(v.itemsize, sizeof(itemtp))
self.assertEqual(v.shape, shape)
# XXX Issue #12851: PyCData_NewGetBuffer() must provide strides
# if requested. memoryview currently reconstructs missing
# stride information, so this assert will fail.
# self.assertEqual(v.strides, ())
self.assertEqual(v.strides, stride)

# they are always read/write
self.assertFalse(v.readonly)
Expand All @@ -51,8 +51,47 @@ def test_native_types(self):
n = n * dim
self.assertEqual(n * v.itemsize, len(v.tobytes()))

def test_native_types_shape_strides(self):
# memoryview fills in the shape and the strides which the exporter
# does not provide, and always requests all of them, so check what
# ctypes exports itself.
for tp, fmt, shape, stride, itemtp in native_types:
with self.subTest(tp=tp):
v = _testbuffer.ndarray(tp(), getbuf=_testbuffer.PyBUF_FULL_RO)
self.assertEqual(v.shape, shape)
self.assertEqual(v.strides, stride)

def test_flags(self):
ob = (c_int * 3 * 2)()

v = _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_SIMPLE)
# ndim > 1 implies shape != NULL, so a flat buffer is exported.
self.assertEqual(v.ndim, 1)
self.assertEqual(v.shape, ())
self.assertEqual(v.strides, ())

v = _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_ND)
self.assertEqual(v.shape, (2, 3))
self.assertEqual(v.strides, ())

v = _testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_STRIDES)
self.assertEqual(v.shape, (2, 3))
self.assertEqual(v.strides, (12, 4))

def test_fortran_contiguous(self):
# A multidimensional array is C contiguous, but not Fortran
# contiguous. A one-dimensional array is contiguous in both orders.
ob = (c_int * 3 * 2)()
with self.assertRaises(BufferError):
_testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_F_CONTIGUOUS)
_testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_C_CONTIGUOUS)

ob = (c_int * 3)()
_testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_F_CONTIGUOUS)
_testbuffer.ndarray(ob, getbuf=_testbuffer.PyBUF_C_CONTIGUOUS)

def test_endian_types(self):
for tp, fmt, shape, itemtp in endian_types:
for tp, fmt, shape, stride, itemtp in endian_types:
ob = tp()
v = memoryview(ob)
self.assertEqual(v.format, fmt)
Expand All @@ -62,8 +101,7 @@ def test_endian_types(self):
self.assertRaises(TypeError, len, v)
self.assertEqual(v.itemsize, sizeof(itemtp))
self.assertEqual(v.shape, shape)
# XXX Issue #12851
# self.assertEqual(v.strides, ())
self.assertEqual(v.strides, stride)

# they are always read/write
self.assertFalse(v.readonly)
Expand Down Expand Up @@ -156,78 +194,78 @@ class Complete(Structure):


native_types = [
# type format shape calc itemsize
# type format shape stride calc itemsize

## simple types

(c_char, "<c", (), c_char),
(c_byte, "<b", (), c_byte),
(c_ubyte, "<B", (), c_ubyte),
(c_short, "<" + s_short, (), c_short),
(c_ushort, "<" + s_ushort, (), c_ushort),
(c_char, "<c", (), (), c_char),
(c_byte, "<b", (), (), c_byte),
(c_ubyte, "<B", (), (), c_ubyte),
(c_short, "<" + s_short, (), (), c_short),
(c_ushort, "<" + s_ushort, (), (), c_ushort),

(c_int, "<" + s_int, (), c_int),
(c_uint, "<" + s_uint, (), c_uint),
(c_int, "<" + s_int, (), (), c_int),
(c_uint, "<" + s_uint, (), (), c_uint),

(c_long, "<" + s_long, (), c_long),
(c_ulong, "<" + s_ulong, (), c_ulong),
(c_long, "<" + s_long, (), (), c_long),
(c_ulong, "<" + s_ulong, (), (), c_ulong),

(c_longlong, "<" + s_longlong, (), c_longlong),
(c_ulonglong, "<" + s_ulonglong, (), c_ulonglong),
(c_longlong, "<" + s_longlong, (), (), c_longlong),
(c_ulonglong, "<" + s_ulonglong, (), (), c_ulonglong),

(c_float, "<f", (), c_float),
(c_double, "<d", (), c_double),
(c_float, "<f", (), (), c_float),
(c_double, "<d", (), (), c_double),

(c_longdouble, "<" + s_longdouble, (), c_longdouble),
(c_longdouble, "<" + s_longdouble, (), (), c_longdouble),

(c_bool, "<" + s_bool, (), c_bool),
(py_object, "<O", (), py_object),
(c_bool, "<" + s_bool, (), (), c_bool),
(py_object, "<O", (), (), py_object),

## pointers

(POINTER(c_byte), "&<b", (), POINTER(c_byte)),
(POINTER(POINTER(c_long)), "&&<" + s_long, (), POINTER(POINTER(c_long))),
(POINTER(c_byte), "&<b", (), (), POINTER(c_byte)),
(POINTER(POINTER(c_long)), "&&<" + s_long, (), (), POINTER(POINTER(c_long))),

## arrays and pointers

(c_double * 4, "<d", (4,), c_double),
(c_double * 0, "<d", (0,), c_double),
(c_float * 4 * 3 * 2, "<f", (2,3,4), c_float),
(c_float * 4 * 0 * 2, "<f", (2,0,4), c_float),
(POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)),
(POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)),
(POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)),
(c_double * 4, "<d", (4,), (8,), c_double),
(c_double * 0, "<d", (0,), (8,), c_double),
(c_float * 4 * 3 * 2, "<f", (2,3,4), (48, 16, 4), c_float),
(c_float * 4 * 0 * 2, "<f", (2,0,4), (0, 16, 4), c_float),
(POINTER(c_short) * 2, "&<" + s_short, (2,), (8,), POINTER(c_short)),
(POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), (16, 8), POINTER(c_short)),
(POINTER(c_short * 2), "&(2)<" + s_short, (), (), POINTER(c_short)),

## structures and unions

(Point2, "T{<l:x:<l:y:}".replace('l', s_long), (), Point2),
(Point, "T{<l:x:<l:y:}".replace('l', s_long), (), Point),
(PackedPoint, "T{<l:x:<l:y:}".replace('l', s_long), (), PackedPoint),
(PointMidPad, "T{<b:x:3x<I:y:}".replace('I', s_uint), (), PointMidPad),
(PackedPointMidPad, "T{<b:x:x<Q:y:}", (), PackedPointMidPad),
(PointEndPad, "T{<I:x:<b:y:3x}".replace('I', s_uint), (), PointEndPad),
(PackedPointEndPad, "T{<Q:x:<b:y:x}", (), PackedPointEndPad),
(EmptyStruct, "T{}", (), EmptyStruct),
(Point2, "T{<l:x:<l:y:}".replace('l', s_long), (), (), Point2),
(Point, "T{<l:x:<l:y:}".replace('l', s_long), (), (), Point),
(PackedPoint, "T{<l:x:<l:y:}".replace('l', s_long), (), (), PackedPoint),
(PointMidPad, "T{<b:x:3x<I:y:}".replace('I', s_uint), (), (), PointMidPad),
(PackedPointMidPad, "T{<b:x:x<Q:y:}", (), (), PackedPointMidPad),
(PointEndPad, "T{<I:x:<b:y:3x}".replace('I', s_uint), (), (), PointEndPad),
(PackedPointEndPad, "T{<Q:x:<b:y:x}", (), (), PackedPointEndPad),
(EmptyStruct, "T{}", (), (), EmptyStruct),
# the pep doesn't support unions
(aUnion, "B", (), aUnion),
(aUnion, "B", (), (), aUnion),
# structure with sub-arrays
(StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (), StructWithArrays),
(StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),
(StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (), (), StructWithArrays),
(StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), (112,), StructWithArrays),

## pointer to incomplete structure
(Incomplete, "B", (), Incomplete),
(POINTER(Incomplete), "&B", (), POINTER(Incomplete)),
(Incomplete, "B", (), (), Incomplete),
(POINTER(Incomplete), "&B", (), (), POINTER(Incomplete)),

# 'Complete' is a structure that starts incomplete, but is completed after the
# pointer type to it has been created.
(Complete, "T{<l:a:}".replace('l', s_long), (), Complete),
(Complete, "T{<l:a:}".replace('l', s_long), (), (), Complete),
# Unfortunately the pointer format string is not fixed...
(POINTER(Complete), "&B", (), POINTER(Complete)),
(POINTER(Complete), "&B", (), (), POINTER(Complete)),

## other

# function signatures are not implemented
(CFUNCTYPE(None), "X{}", (), CFUNCTYPE(None)),
(CFUNCTYPE(None), "X{}", (), (), CFUNCTYPE(None)),

]

Expand All @@ -242,10 +280,10 @@ class LEPoint(LittleEndianStructure):
# This table contains format strings as they really look, on both big
# and little endian machines.
endian_types = [
(BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), (), BEPoint),
(LEPoint * 1, "T{<l:x:<l:y:}".replace('l', s_long), (1,), LEPoint),
(POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), (), POINTER(BEPoint)),
(POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), (), POINTER(LEPoint)),
(BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), (), (), BEPoint),
(LEPoint * 1, "T{<l:x:<l:y:}".replace('l', s_long), (1,), (16,), LEPoint),
(POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), (), (), POINTER(BEPoint)),
(POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), (), (), POINTER(LEPoint)),
]


Expand Down
4 changes: 4 additions & 0 deletions Misc/NEWS.d/next/Library/2017-12-08-19-58-16.gh-issue-57060.5H37Gx.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
The buffer exported by :mod:`ctypes` arrays now provides strides if they are
requested, as required by :pep:`3118`. The format and the shape are now
provided only if they are requested, and requesting a Fortran contiguous
buffer for a multidimensional array now fails.
45 changes: 39 additions & 6 deletions Modules/_ctypes/_ctypes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -502,8 +502,10 @@ ctype_free_stginfo_members(StgInfo *info)
info->ffi_type_pointer.elements = NULL;
PyMem_Free(info->format);
info->format = NULL;
/* The strides point into the shape allocation. */
PyMem_Free(info->shape);
info->shape = NULL;
info->strides = NULL;
ctype_clear_stginfo(info);
}

Expand Down Expand Up @@ -1785,7 +1787,7 @@ PyCArrayType_init(PyObject *self, PyObject *args, PyObject *kwds)
if (stginfo->format == NULL)
goto error;
stginfo->ndim = iteminfo->ndim + 1;
stginfo->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stginfo->ndim);
stginfo->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stginfo->ndim * 2);
if (stginfo->shape == NULL) {
PyErr_NoMemory();
goto error;
Expand All @@ -1795,6 +1797,15 @@ PyCArrayType_init(PyObject *self, PyObject *args, PyObject *kwds)
memmove(&stginfo->shape[1], iteminfo->shape,
sizeof(Py_ssize_t) * (stginfo->ndim - 1));
}
stginfo->strides = stginfo->shape + stginfo->ndim;
if (stginfo->ndim > 1) {
memmove(&stginfo->strides[1], iteminfo->strides,
sizeof(Py_ssize_t) * (stginfo->ndim - 1));
stginfo->strides[0] = stginfo->strides[1] * stginfo->shape[1];
}
else {
stginfo->strides[0] = iteminfo->size;
}

itemsize = iteminfo->size;
if (itemsize != 0 && length > PY_SSIZE_T_MAX / itemsize) {
Expand Down Expand Up @@ -3125,16 +3136,38 @@ PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags)
}
assert(item_info);

if (info->ndim > 1 && (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) {
PyErr_SetString(PyExc_BufferError,
"ctypes array is not Fortran contiguous");
return -1;
}

view->buf = self->b_ptr;
view->obj = Py_NewRef(myself);
view->len = self->b_size;
view->readonly = 0;
/* use default format character if not set */
view->format = info->format ? info->format : "B";
view->ndim = info->ndim;
view->shape = info->shape;
if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) {
/* use default format character if not set */
view->format = info->format ? info->format : "B";
}
else {
view->format = NULL;
}
if ((flags & PyBUF_ND) == PyBUF_ND) {
view->ndim = info->ndim;
view->shape = info->shape;
}
else {
/* The buffer is C contiguous, so it can be exposed as flat.
Keep ndim <= 1: ndim > 1 implies shape != NULL, see
PyBuffer_IsContiguous(). */
view->ndim = info->ndim ? 1 : 0;
view->shape = NULL;
}
view->itemsize = item_info->size;
view->strides = NULL;
/* PyBUF_STRIDES implies PyBUF_ND. */
view->strides = ((flags & PyBUF_STRIDES) == PyBUF_STRIDES)
? info->strides : NULL;
view->suboffsets = NULL;
view->internal = NULL;
return 0;
Expand Down
2 changes: 1 addition & 1 deletion Modules/_ctypes/ctypes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ typedef struct {
char *format;
int ndim;
Py_ssize_t *shape;
/* Py_ssize_t *strides; */ /* unused in ctypes */
Py_ssize_t *strides; /* offset from *shape, not necessary to PyMem_Free */
/* Py_ssize_t *suboffsets; */ /* unused in ctypes */
} StgInfo;

Expand Down
9 changes: 8 additions & 1 deletion Modules/_ctypes/stgdict.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,21 @@ PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info)
}
strcpy(dst_info->format, src_info->format);
}
dst_info->strides = NULL;
if (src_info->shape) {
dst_info->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src_info->ndim);
dst_info->shape = PyMem_Malloc(
sizeof(Py_ssize_t) * src_info->ndim * 2);
if (dst_info->shape == NULL) {
PyErr_NoMemory();
return -1;
}
memcpy(dst_info->shape, src_info->shape,
sizeof(Py_ssize_t) * src_info->ndim);
if (src_info->strides) {
dst_info->strides = dst_info->shape + src_info->ndim;
memcpy(dst_info->strides, src_info->strides,
sizeof(Py_ssize_t) * src_info->ndim);
}
}

if (src_info->ffi_type_pointer.elements == NULL)
Expand Down
Loading