Skip to content

gh-112014: correct buffer protocol support in ctypes (use native formats) #155472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
skirpichev wants to merge 3 commits into
base: main
Choose a base branch
from
+84 −83
Draft

gh-112014: correct buffer protocol support in ctypes (use native formats) #155472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
feb9f5a
gh-112014: correct buffer protocol support in ctypes (use native form…
skirpichev Aug 10, 2026
b82ee36
Merge branch 'master' into use-native-endianness-in-ctypes/112014
skirpichev Aug 11, 2026
a58b4e4
address review: use native types
skirpichev Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions Lib/test/test_ctypes/test_numbers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,10 @@ def test_complex(self):
self.assertEqual(t(FloatLike()).value, 2+0j)
self.assertEqual(t(ComplexLike()).value, 1+1j)

prefix = '>' if sys.byteorder == 'big' else '<'
num = t(1.0)
self.assertEqual(memoryview(num).format, prefix + format)
self.assertEqual(memoryview(num).format, format)
array = (t * 3)()
self.assertEqual(memoryview(array).format, prefix + format)
self.assertEqual(memoryview(array).format, format)

@unittest.skipUnless(hasattr(ctypes, "c_double_complex"),
"requires C11 complex type")
Expand Down
147 changes: 74 additions & 73 deletions Lib/test/test_ctypes/test_pep3118.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import re
import sys
import unittest
from ctypes import (CFUNCTYPE, POINTER, sizeof, Union,
Expand All @@ -9,29 +8,12 @@
c_bool, c_float, c_double, c_longdouble, py_object)


if sys.byteorder == "little":
THIS_ENDIAN = "<"
OTHER_ENDIAN = ">"
else:
THIS_ENDIAN = ">"
OTHER_ENDIAN = "<"


def normalize(format):
# Remove current endian specifier and white space from a format
# string
if format is None:
return ""
format = format.replace(OTHER_ENDIAN, THIS_ENDIAN)
return re.sub(r"\s", "", format)


class Test(unittest.TestCase):
def test_native_types(self):
for tp, fmt, shape, itemtp in native_types:
ob = tp()
v = memoryview(ob)
self.assertEqual(normalize(v.format), normalize(fmt))
self.assertEqual(v.format, fmt)
if shape:
self.assertEqual(len(v), shape[0])
else:
Expand Down Expand Up @@ -73,6 +55,15 @@ def test_endian_types(self):
n = n * dim
self.assertEqual(n * v.itemsize, len(v.tobytes()))

def test_memoryview_supports_ctypes_arrays(self):
ArrayType = c_int * 5
a = ArrayType(123, 42, 1, 2, 3)
m = memoryview(a)
self.assertEqual(m.shape, (5,))
self.assertEqual(m.format, c_int._type_)
self.assertEqual(list(m), [123, 42, 1, 2, 3])
self.assertEqual(m[1], 42)


# define some structure classes

Expand Down Expand Up @@ -124,20 +115,19 @@ class Complete(Structure):

################################################################
#
# This table contains format strings as they look on little endian
# machines. The test replaces '<' with '>' on big endian machines.
# This table contains format strings with native endianness.
#

# Platform-specific type codes
s_bool = {1: '?', 2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_bool)]
s_short = {2: 'h', 4: 'l', 8: 'q'}[sizeof(c_short)]
s_ushort = {2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_ushort)]
s_int = {2: 'h', 4: 'i', 8: 'q'}[sizeof(c_int)]
s_uint = {2: 'H', 4: 'I', 8: 'Q'}[sizeof(c_uint)]
s_long = {4: 'l', 8: 'q'}[sizeof(c_long)]
s_ulong = {4: 'L', 8: 'Q'}[sizeof(c_ulong)]
s_longlong = "q"
s_ulonglong = "Q"
s_bool = c_bool._type_
s_short = c_short._type_
s_ushort = c_ushort._type_
s_int = c_int._type_
s_uint = c_uint._type_
s_long = c_long._type_
s_ulong = c_ulong._type_
s_longlong = c_longlong._type_
s_ulonglong = c_ulonglong._type_
s_float = "f"
s_double = "d"
s_longdouble = "g"
Expand All @@ -160,67 +150,67 @@ class Complete(Structure):

## simple types

(c_char, "<c", (), c_char),
(c_byte, "<b", (), c_byte),
(c_ubyte, "<B", (), c_ubyte),
(c_short, "<" + s_short, (), c_short),
(c_ushort, "<" + s_ushort, (), c_ushort),
(c_char, "c", (), c_char),
(c_byte, "b", (), c_byte),
(c_ubyte, "B", (), c_ubyte),
(c_short, s_short, (), c_short),
(c_ushort, s_ushort, (), c_ushort),

(c_int, "<" + s_int, (), c_int),
(c_uint, "<" + s_uint, (), c_uint),
(c_int, s_int, (), c_int),
(c_uint, s_uint, (), c_uint),

(c_long, "<" + s_long, (), c_long),
(c_ulong, "<" + s_ulong, (), c_ulong),
(c_long, s_long, (), c_long),
(c_ulong, s_ulong, (), c_ulong),

(c_longlong, "<" + s_longlong, (), c_longlong),
(c_ulonglong, "<" + s_ulonglong, (), c_ulonglong),
(c_longlong, s_longlong, (), c_longlong),
(c_ulonglong, s_ulonglong, (), c_ulonglong),

(c_float, "<f", (), c_float),
(c_double, "<d", (), c_double),
(c_float, "f", (), c_float),
(c_double, "d", (), c_double),

(c_longdouble, "<" + s_longdouble, (), c_longdouble),
(c_longdouble, s_longdouble, (), c_longdouble),

(c_bool, "<" + s_bool, (), c_bool),
(py_object, "<O", (), py_object),
(c_bool, s_bool, (), c_bool),
(py_object, "O", (), py_object),

## pointers

(POINTER(c_byte), "&<b", (), POINTER(c_byte)),
(POINTER(POINTER(c_long)), "&&<" + s_long, (), POINTER(POINTER(c_long))),
(POINTER(c_byte), "&b", (), POINTER(c_byte)),
(POINTER(POINTER(c_long)), "&&" + s_long, (), POINTER(POINTER(c_long))),

## arrays and pointers

(c_double * 4, "<d", (4,), c_double),
(c_double * 0, "<d", (0,), c_double),
(c_float * 4 * 3 * 2, "<f", (2,3,4), c_float),
(c_float * 4 * 0 * 2, "<f", (2,0,4), c_float),
(POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)),
(POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)),
(POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)),
(c_double * 4, "d", (4,), c_double),
(c_double * 0, "d", (0,), c_double),
(c_float * 4 * 3 * 2, "f", (2,3,4), c_float),
(c_float * 4 * 0 * 2, "f", (2,0,4), c_float),
(POINTER(c_short) * 2, "&" + s_short, (2,), POINTER(c_short)),
(POINTER(c_short) * 2 * 3, "&" + s_short, (3,2,), POINTER(c_short)),
(POINTER(c_short * 2), "&(2)" + s_short, (), POINTER(c_short)),

## structures and unions

(Point2, "T{<l:x:<l:y:}".replace('l', s_long), (), Point2),
(Point, "T{<l:x:<l:y:}".replace('l', s_long), (), Point),
(PackedPoint, "T{<l:x:<l:y:}".replace('l', s_long), (), PackedPoint),
(PointMidPad, "T{<b:x:3x<I:y:}".replace('I', s_uint), (), PointMidPad),
(PackedPointMidPad, "T{<b:x:x<Q:y:}", (), PackedPointMidPad),
(PointEndPad, "T{<I:x:<b:y:3x}".replace('I', s_uint), (), PointEndPad),
(PackedPointEndPad, "T{<Q:x:<b:y:x}", (), PackedPointEndPad),
(EmptyStruct, "T{}", (), EmptyStruct),
(Point2, "T{l:x:l:y:}".replace('l', s_long), (), Point2),
(Point, "T{l:x:l:y:}".replace('l', s_long), (), Point),
(PackedPoint, "T{l:x:l:y:}".replace('l', s_long), (), PackedPoint),
(PointMidPad, "T{b:x:3xI:y:}".replace('I', s_uint), (), PointMidPad),
(PackedPointMidPad, "T{b:x:xQ:y:}".replace('Q', s_ulonglong), (), PackedPointMidPad),
(PointEndPad, "T{I:x:b:y:3x}".replace('I', s_uint), (), PointEndPad),
(PackedPointEndPad, "T{Q:x:b:y:x}".replace('Q', s_ulonglong), (), PackedPointEndPad),
(EmptyStruct, "T{}", (), EmptyStruct),
# the pep doesn't support unions
(aUnion, "B", (), aUnion),
(aUnion, "B", (), aUnion),
# structure with sub-arrays
(StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (), StructWithArrays),
(StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),
(StructWithArrays, "T{(2,3)l:x:(4)T{l:x:l:y:}:y:}".replace('l', s_long), (), StructWithArrays),
(StructWithArrays * 3, "T{(2,3)l:x:(4)T{l:x:l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),

## pointer to incomplete structure
(Incomplete, "B", (), Incomplete),
(POINTER(Incomplete), "&B", (), POINTER(Incomplete)),

# 'Complete' is a structure that starts incomplete, but is completed after the
# pointer type to it has been created.
(Complete, "T{<l:a:}".replace('l', s_long), (), Complete),
(Complete, "T{l:a:}".replace('l', s_long), (), Complete),
# Unfortunately the pointer format string is not fixed...
(POINTER(Complete), "&B", (), POINTER(Complete)),

Expand All @@ -239,14 +229,25 @@ class LEPoint(LittleEndianStructure):
_fields_ = [("x", c_long), ("y", c_long)]


s_long2 = {4: 'l', 8: 'q'}[sizeof(c_long)]


# This table contains format strings as they really look, on both big
# and little endian machines.
endian_types = [
(BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), (), BEPoint),
(LEPoint * 1, "T{<l:x:<l:y:}".replace('l', s_long), (1,), LEPoint),
(POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), (), POINTER(BEPoint)),
(POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), (), POINTER(LEPoint)),
]
if sys.byteorder == "little":
endian_types = [
(BEPoint, "T{>l:x:>l:y:}".replace('l', s_long2), (), BEPoint),
(LEPoint * 1, "T{l:x:l:y:}", (1,), LEPoint),
(POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long2), (), POINTER(BEPoint)),
(POINTER(LEPoint), "&T{l:x:l:y:}", (), POINTER(LEPoint)),
]
else:
endian_types = [
(BEPoint * 1, "T{l:x:l:y:}", (1,), BEPoint),
(LEPoint, "T{<l:x:<l:y:}".replace('l', s_long2), (), LEPoint),
(POINTER(BEPoint), "&T{l:x:l:y:}", (), POINTER(BEPoint)),
(POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long2), (), POINTER(LEPoint)),
]


if __name__ == "__main__":
Expand Down
4 changes: 4 additions & 0 deletions Misc/NEWS.d/next/Library/2026-08-10-10-35-19.gh-issue-112014.nLEBCv.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Correct the :ref:`buffer protocol <bufferobjects>` support in the
:mod:`ctypes` module to use the machine’s native format and byte order,
rather than explicitly specify endianness (by ``'<'`` or ``'>'``). The
later kept for byte-swapped types. Patch by Sergey B Kirpichev.

@vstinner vstinner Aug 10, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From this NEWS entry, it's uneasy to understand the rationale for the change. According to #112014 (comment), I understand that before, it was not possible to modify a ctypes type using memoryview, since the buffer format was rejected by memoryview when attempting to modify the view.

@skirpichev skirpichev Aug 10, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's uneasy to understand the rationale for the change

It's a bug :-) Memoryview support for ctypes buffers is a side effect of the bugfix.

11 changes: 4 additions & 7 deletions Modules/_ctypes/_ctypes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2405,15 +2405,12 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds)
stginfo->size = fmt->pffi_type->size;
stginfo->setfunc = fmt->setfunc;
stginfo->getfunc = fmt->getfunc;
#ifdef WORDS_BIGENDIAN
stginfo->format = _ctypes_alloc_format_string_for_type(proto_str, 1);
#else
stginfo->format = _ctypes_alloc_format_string_for_type(proto_str, 0);
#endif
stginfo->format = PyMem_Malloc(1 + strlen(proto_str));
if (stginfo->format == NULL) {
Py_DECREF(proto);
return -1;
}
strcpy(stginfo->format, proto_str);

stginfo->paramfunc = PyCSimpleType_paramfunc;
/*
Expand Down Expand Up @@ -2504,14 +2501,14 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds)
PyObject_SetAttrString(swapped, "__ctype_be__", self);
PyObject_SetAttrString(swapped, "__ctype_le__", swapped);
/* We are creating the type for the OTHER endian */
sw_info->format = _ctypes_alloc_format_string("<", stginfo->format+1);
sw_info->format = _ctypes_alloc_format_string_for_type(stginfo->format, 0);
#else
PyObject_SetAttrString(self, "__ctype_be__", swapped);
PyObject_SetAttrString(self, "__ctype_le__", self);
PyObject_SetAttrString(swapped, "__ctype_le__", self);
PyObject_SetAttrString(swapped, "__ctype_be__", swapped);
/* We are creating the type for the OTHER endian */
sw_info->format = _ctypes_alloc_format_string(">", stginfo->format+1);
sw_info->format = _ctypes_alloc_format_string_for_type(stginfo->format, 1);
#endif
Py_DECREF(swapped);
if (PyErr_Occurred()) {
Expand Down
Loading