diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 758415a76e5cb4..8f560699d355e4 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -74,11 +74,25 @@ List Objects Like :c:func:`PyList_GetItemRef`, but returns a :term:`borrowed reference` instead of a :term:`strong reference`. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) Similar to :c:func:`PyList_GetItem`, but without error checking. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) @@ -108,6 +122,14 @@ List Objects is being replaced; any reference in *list* at position *i* will be leaked. + .. note:: + + In the :term:`free-threaded build`, this macro has no internal + synchronization. It is normally only used to fill in new lists where no + other thread has a reference to the list. If the list may be shared, + use :c:func:`PyList_SetItem` instead, which uses a :term:`per-object + lock`. + .. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item) @@ -138,6 +160,12 @@ List Objects Return ``0`` on success, ``-1`` on failure. Indexing from the end of the list is not supported. + .. note:: + + In the :term:`free-threaded build`, when *itemlist* is a :class:`list`, + both *list* and *itemlist* are locked for the duration of the operation. + For other iterables (or ``NULL``), only *list* is locked. + .. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable) @@ -150,6 +178,14 @@ List Objects .. versionadded:: 3.13 + .. note:: + + In the :term:`free-threaded build`, when *iterable* is a :class:`list`, + :class:`set`, :class:`dict`, or dict view, both *list* and *iterable* + (or its underlying dict) are locked for the duration of the operation. + For other iterables, only *list* is locked; *iterable* may be + concurrently modified by another thread. + .. c:function:: int PyList_Clear(PyObject *list) @@ -168,6 +204,14 @@ List Objects Sort the items of *list* in place. Return ``0`` on success, ``-1`` on failure. This is equivalent to ``list.sort()``. + .. note:: + + In the :term:`free-threaded build`, element comparison via + :meth:`~object.__lt__` can execute arbitrary Python code, during which + the :term:`per-object lock` may be temporarily released. For built-in + types (:class:`str`, :class:`int`, :class:`float`), the lock is not + released during comparison. + .. c:function:: int PyList_Reverse(PyObject *list) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 87b488912653b9..cd13c0f4d61a42 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -3057,6 +3057,24 @@ Buffer Object Structures (5) Return ``0``. + **Thread safety:** + + In the :term:`free-threaded build`, implementations must ensure: + + * The export counter increment in step (3) is atomic. + + * The underlying buffer data remains valid and at a stable memory + location for the lifetime of all exports. + + * For objects that support resizing or reallocation (such as + :class:`bytearray`), the export counter is checked atomically before + such operations, and :exc:`BufferError` is raised if exports exist. + + * The function is safe to call concurrently from multiple threads. + + See also :ref:`thread-safety-memoryview` for the Python-level + thread safety guarantees of :class:`memoryview` objects. + If *exporter* is part of a chain or tree of buffer providers, two main schemes can be used: @@ -3102,6 +3120,16 @@ Buffer Object Structures (2) If the counter is ``0``, free all memory associated with *view*. + **Thread safety:** + + In the :term:`free-threaded build`: + + * The export counter decrement in step (1) must be atomic. + + * Resource cleanup when the counter reaches zero must be done atomically, + as the final release may race with concurrent releases from other + threads and dellocation must only happen once. + The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep track of buffer-specific resources. This field is guaranteed to remain constant, while a consumer MAY pass a copy of the original buffer as the diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat index f063ca1360d5fb..103e8ef3e97ed1 100644 --- a/Doc/data/threadsafety.dat +++ b/Doc/data/threadsafety.dat @@ -17,3 +17,59 @@ PyMutex_Lock:shared: PyMutex_Unlock:shared: PyMutex_IsLocked:atomic: + +# List objects (Doc/c-api/list.rst) + +# Type checks - read ob_type pointer, always safe +PyList_Check:atomic: +PyList_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyList_New:atomic: + +# Size - uses atomic load on free-threaded builds +PyList_Size:atomic: +PyList_GET_SIZE:atomic: + +# Strong-reference lookup - lock-free with atomic ops +PyList_GetItemRef:atomic: + +# Borrowed-reference lookups - no locking; returned borrowed +# reference is unsafe in free-threaded builds without +# external synchronization +PyList_GetItem:compatible: +PyList_GET_ITEM:compatible: + +# Single-item mutations - hold per-object lock for duration; +# appear atomic to lock-free readers +PyList_SetItem:atomic: +PyList_Append:atomic: + +# Insert - protected by per-object critical section; shifts +# elements so lock-free readers may observe intermediate states +PyList_Insert:shared: + +# Initialization macro - no synchronization; normally only used +# to fill in new lists where there is no previous content +PyList_SET_ITEM:compatible: + +# Bulk operations - hold per-object lock for duration +PyList_GetSlice:atomic: +PyList_AsTuple:atomic: +PyList_Clear:atomic: + +# Reverse - protected by per-object critical section; swaps +# elements so lock-free readers may observe intermediate states +PyList_Reverse:shared: + +# Slice assignment - lock target list; also lock source when it +# is a list +PyList_SetSlice:shared: + +# Sort - per-object lock held; comparison callbacks may execute +# arbitrary Python code +PyList_Sort:shared: + +# Extend - lock target list; also lock source when it is a +# list, set, or dict +PyList_Extend:shared: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 24f53a3a272d73..48291622d7be89 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5045,6 +5045,9 @@ copying. .. versionadded:: 3.3 +For information on the thread safety of :class:`memoryview` objects in +the :term:`free-threaded build`, see :ref:`thread-safety-memoryview`. + .. _types-set: diff --git a/Doc/library/threadsafety.rst b/Doc/library/threadsafety.rst index 8063c2ea5011e7..a529f7803affbc 100644 --- a/Doc/library/threadsafety.rst +++ b/Doc/library/threadsafety.rst @@ -548,3 +548,59 @@ Thread safety for bytearray objects Consider external synchronization when sharing :class:`bytearray` instances across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-memoryview: + +Thread safety for memoryview objects +==================================== + +:class:`memoryview` objects provide access to the internal data of an +underlying object without copying. Thread safety depends on both the +memoryview itself and the underlying buffer exporter. + +The memoryview implementation uses atomic operations to track its own +exports in the :term:`free-threaded build`. Creating and +releasing a memoryview are thread-safe. Attribute access (e.g., +:attr:`~memoryview.shape`, :attr:`~memoryview.format`) reads fields that +are immutable for the lifetime of the memoryview, so concurrent reads +are safe as long as the memoryview has not been released. + +However, the actual data accessed through the memoryview is owned by the +underlying object. Concurrent access to this data is only safe if the +underlying object supports it: + +* For immutable objects like :class:`bytes`, concurrent reads through + multiple memoryviews are safe. + +* For mutable objects like :class:`bytearray`, reading and writing the + same memory region from multiple threads without external + synchronization is not safe and may result in data corruption. + Note that even read-only memoryviews of mutable objects do not + prevent data races if the underlying object is modified from + another thread. + +.. code-block:: + :class: bad + + # NOT safe: concurrent writes to the same buffer + data = bytearray(1000) + view = memoryview(data) + # Thread 1: view[0:500] = b'x' * 500 + # Thread 2: view[0:500] = b'y' * 500 + +.. code-block:: + :class: good + + # Safe: use a lock for concurrent access + import threading + lock = threading.Lock() + data = bytearray(1000) + view = memoryview(data) + + with lock: + view[0:500] = b'x' * 500 + +Resizing or reallocating the underlying object (such as calling +:meth:`bytearray.resize`) while a memoryview is exported raises +:exc:`BufferError`. This is enforced regardless of threading. diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index ff020b52da3f23..9d30a14f112937 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -9,14 +9,19 @@ -------------- The :mod:`!wave` module provides a convenient interface to the Waveform Audio -"WAVE" (or "WAV") file format. Only uncompressed PCM encoded wave files are -supported. +"WAVE" (or "WAV") file format. + +The module supports uncompressed PCM and IEEE floating-point WAV formats. .. versionchanged:: 3.12 Support for ``WAVE_FORMAT_EXTENSIBLE`` headers was added, provided that the extended format is ``KSDATAFORMAT_SUBTYPE_PCM``. +.. versionchanged:: next + + Support for reading and writing ``WAVE_FORMAT_IEEE_FLOAT`` files was added. + The :mod:`!wave` module defines the following function and exception: @@ -60,6 +65,21 @@ The :mod:`!wave` module defines the following function and exception: specification or hits an implementation deficiency. +.. data:: WAVE_FORMAT_PCM + + Format code for uncompressed PCM audio. + + +.. data:: WAVE_FORMAT_IEEE_FLOAT + + Format code for IEEE floating-point audio. + + +.. data:: WAVE_FORMAT_EXTENSIBLE + + Format code for WAVE extensible headers. + + .. _wave-read-objects: Wave_read Objects @@ -98,6 +118,14 @@ Wave_read Objects Returns number of audio frames. + .. method:: getformat() + + Returns the frame format code. + + This is one of :data:`WAVE_FORMAT_PCM`, + :data:`WAVE_FORMAT_IEEE_FLOAT`, or :data:`WAVE_FORMAT_EXTENSIBLE`. + + .. method:: getcomptype() Returns compression type (``'NONE'`` is the only supported type). @@ -112,8 +140,8 @@ Wave_read Objects .. method:: getparams() Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - ``get*()`` methods. + framerate, nframes, comptype, compname)``, equivalent to output + of the ``get*()`` methods. .. method:: readframes(n) @@ -190,6 +218,9 @@ Wave_write Objects Set the sample width to *n* bytes. + For :data:`WAVE_FORMAT_IEEE_FLOAT`, only 4-byte (32-bit) and + 8-byte (64-bit) sample widths are supported. + .. method:: getsampwidth() @@ -238,11 +269,32 @@ Wave_write Objects Return the human-readable compression type name. + .. method:: setformat(format) + + Set the frame format code. + + Supported values are :data:`WAVE_FORMAT_PCM` and + :data:`WAVE_FORMAT_IEEE_FLOAT`. + + When setting :data:`WAVE_FORMAT_IEEE_FLOAT`, the sample width must be + 4 or 8 bytes. + + + .. method:: getformat() + + Return the current frame format code. + + .. method:: setparams(tuple) - The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, with values valid for the ``set*()`` methods. Sets all - parameters. + The *tuple* should be + ``(nchannels, sampwidth, framerate, nframes, comptype, compname, format)``, + with values valid for the ``set*()`` methods. Sets all parameters. + + For backwards compatibility, a 6-item tuple without *format* is also + accepted and defaults to :data:`WAVE_FORMAT_PCM`. + + For ``format=WAVE_FORMAT_IEEE_FLOAT``, *sampwidth* must be 4 or 8. .. method:: getparams() @@ -279,3 +331,6 @@ Wave_write Objects Note that it is invalid to set any parameters after calling :meth:`writeframes` or :meth:`writeframesraw`, and any attempt to do so will raise :exc:`wave.Error`. + + For :data:`WAVE_FORMAT_IEEE_FLOAT` output, a ``fact`` chunk is written as + required by the WAVE specification for non-PCM formats. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 90b8821daaf3fb..1e53c0e0e6f971 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -3637,12 +3637,25 @@ implement the protocol in Python. provides a convenient way to interpret the flags. The method must return a :class:`memoryview` object. + **Thread safety:** In :term:`free-threaded ` Python, + implementations must manage any internal export counter using atomic + operations. The method must be safe to call concurrently from multiple + threads, and the returned buffer's underlying data must remain valid + until the corresponding :meth:`~object.__release_buffer__` call + completes. See :ref:`thread-safety-memoryview` for details. + .. method:: object.__release_buffer__(self, buffer) Called when a buffer is no longer needed. The *buffer* argument is a :class:`memoryview` object that was previously returned by :meth:`~object.__buffer__`. The method must release any resources associated with the buffer. This method should return ``None``. + + **Thread safety:** In :term:`free-threaded ` Python, + any export counter decrement must use atomic operations. Resource + cleanup must be thread-safe, as the final release may race with + concurrent releases from other threads. + Buffer objects that do not need to perform any cleanup are not required to implement this method. diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index ae21dfdbf0ac44..3c6edf2c4793ab 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -549,9 +549,9 @@ caught like any other exception. :: >>> try: ... f() ... except Exception as e: - ... print(f'caught {type(e)}: e') + ... print(f'caught {type(e)}: {e}') ... - caught : e + caught : there were problems (2 sub-exceptions) >>> By using ``except*`` instead of ``except``, we can selectively diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index d4c4483bc5eb78..7d5c6e224789be 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1511,6 +1511,21 @@ typing wave ---- +* Added support for IEEE floating-point WAVE audio + (``WAVE_FORMAT_IEEE_FLOAT``) in :mod:`wave`. + +* Added :meth:`wave.Wave_read.getformat`, :meth:`wave.Wave_write.getformat`, + and :meth:`wave.Wave_write.setformat` for explicit frame format handling. + +* :meth:`wave.Wave_write.setparams` accepts both 7-item tuples including + ``format`` and 6-item tuples for backwards compatibility (defaulting to + ``WAVE_FORMAT_PCM``). + +* ``WAVE_FORMAT_IEEE_FLOAT`` output now includes a ``fact`` chunk, + as required for non-PCM WAVE formats. + +(Contributed by Lionel Koenig and Michiel W. Beijen in :gh:`60729`.) + * Removed the ``getmark()``, ``setmark()`` and ``getmarkers()`` methods of the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes, which were deprecated since Python 3.13. diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8b52d77538abf2..6b3c39d64c9c32 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -159,10 +159,87 @@ PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); int _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorDetach(_PyExecutorObject *); -void _Py_BloomFilter_Init(_PyBloomFilter *); -void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); +/* We use a bloomfilter with k = 6, m = 256 + * The choice of k and the following constants + * could do with a more rigorous analysis, + * but here is a simple analysis: + * + * We want to keep the false positive rate low. + * For n = 5 (a trace depends on 5 objects), + * we expect 30 bits set, giving a false positive + * rate of (30/256)**6 == 2.5e-6 which is plenty + * good enough. + * + * However with n = 10 we expect 60 bits set (worst case), + * giving a false positive of (60/256)**6 == 0.0001 + * + * We choose k = 6, rather than a higher number as + * it means the false positive rate grows slower for high n. + * + * n = 5, k = 6 => fp = 2.6e-6 + * n = 5, k = 8 => fp = 3.5e-7 + * n = 10, k = 6 => fp = 1.6e-4 + * n = 10, k = 8 => fp = 0.9e-4 + * n = 15, k = 6 => fp = 0.18% + * n = 15, k = 8 => fp = 0.23% + * n = 20, k = 6 => fp = 1.1% + * n = 20, k = 8 => fp = 2.3% + * + * The above analysis assumes perfect hash functions, + * but those don't exist, so the real false positive + * rates may be worse. + */ + +#define _Py_BLOOM_FILTER_K 6 +#define _Py_BLOOM_FILTER_SEED 20221211 + +static inline uint64_t +address_to_hash(void *ptr) { + assert(ptr != NULL); + uint64_t uhash = _Py_BLOOM_FILTER_SEED; + uintptr_t addr = (uintptr_t)ptr; + for (int i = 0; i < SIZEOF_VOID_P; i++) { + uhash ^= addr & 255; + uhash *= (uint64_t)PyHASH_MULTIPLIER; + addr >>= 8; + } + return uhash; +} + +static inline void +_Py_BloomFilter_Init(_PyBloomFilter *bloom) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + bloom->bits[i] = 0; + } +} + +static inline void +_Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) +{ + uint64_t hash = address_to_hash(ptr); + assert(_Py_BLOOM_FILTER_K <= 8); + for (int i = 0; i < _Py_BLOOM_FILTER_K; i++) { + uint8_t bits = hash & 255; + bloom->bits[bits >> _Py_BLOOM_FILTER_WORD_SHIFT] |= + (_Py_bloom_filter_word_t)1 << (bits & (_Py_BLOOM_FILTER_BITS_PER_WORD - 1)); + hash >>= 8; + } +} + +static inline bool +bloom_filter_may_contain(const _PyBloomFilter *bloom, const _PyBloomFilter *hashes) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { + return false; + } + } + return true; +} + #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index e7ac7d59ff7e27..7bc8947cfa9a9d 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -45,10 +45,21 @@ typedef struct _PyUOpInstruction{ /* Bloom filter with m = 256 * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 +#ifdef HAVE_GCC_UINT128_T +#define _Py_BLOOM_FILTER_WORDS 2 +typedef __uint128_t _Py_bloom_filter_word_t; +#else +#define _Py_BLOOM_FILTER_WORDS 4 +typedef uint64_t _Py_bloom_filter_word_t; +#endif + +#define _Py_BLOOM_FILTER_BITS_PER_WORD \ + ((int)(sizeof(_Py_bloom_filter_word_t) * 8)) +#define _Py_BLOOM_FILTER_WORD_SHIFT \ + ((sizeof(_Py_bloom_filter_word_t) == 16) ? 7 : 6) typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; + _Py_bloom_filter_word_t bits[_Py_BLOOM_FILTER_WORDS]; } _PyBloomFilter; #ifdef __cplusplus diff --git a/Lib/argparse.py b/Lib/argparse.py index 296a210ad832da..d91707d9eec546 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2623,7 +2623,7 @@ def _get_nargs_pattern(self, action): # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([AO]*)' if option else '(.*)' + nargs_pattern = '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: diff --git a/Lib/test/audiodata/pluck-float32.wav b/Lib/test/audiodata/pluck-float32.wav new file mode 100644 index 00000000000000..2030fb16d6e3bd Binary files /dev/null and b/Lib/test/audiodata/pluck-float32.wav differ diff --git a/Lib/test/audiotests.py b/Lib/test/audiotests.py index 9d6c4cc2b4b02c..394097df17dca9 100644 --- a/Lib/test/audiotests.py +++ b/Lib/test/audiotests.py @@ -27,17 +27,18 @@ def tearDown(self): unlink(TESTFN) def check_params(self, f, nchannels, sampwidth, framerate, nframes, - comptype, compname): + comptype, compname, format): self.assertEqual(f.getnchannels(), nchannels) self.assertEqual(f.getsampwidth(), sampwidth) self.assertEqual(f.getframerate(), framerate) self.assertEqual(f.getnframes(), nframes) self.assertEqual(f.getcomptype(), comptype) self.assertEqual(f.getcompname(), compname) + self.assertEqual(f.getformat(), format) params = f.getparams() self.assertEqual(params, - (nchannels, sampwidth, framerate, nframes, comptype, compname)) + (nchannels, sampwidth, framerate, nframes, comptype, compname)) self.assertEqual(params.nchannels, nchannels) self.assertEqual(params.sampwidth, sampwidth) self.assertEqual(params.framerate, framerate) @@ -51,13 +52,17 @@ def check_params(self, f, nchannels, sampwidth, framerate, nframes, class AudioWriteTests(AudioTests): + readonly = False def create_file(self, testfile): + if self.readonly: + self.skipTest('Read only file format') f = self.fout = self.module.open(testfile, 'wb') f.setnchannels(self.nchannels) f.setsampwidth(self.sampwidth) f.setframerate(self.framerate) f.setcomptype(self.comptype, self.compname) + f.setformat(self.format) return f def check_file(self, testfile, nframes, frames): @@ -67,13 +72,14 @@ def check_file(self, testfile, nframes, frames): self.assertEqual(f.getframerate(), self.framerate) self.assertEqual(f.getnframes(), nframes) self.assertEqual(f.readframes(nframes), frames) + self.assertEqual(f.getformat(), self.format) def test_write_params(self): f = self.create_file(TESTFN) f.setnframes(self.nframes) f.writeframes(self.frames) self.check_params(f, self.nchannels, self.sampwidth, self.framerate, - self.nframes, self.comptype, self.compname) + self.nframes, self.comptype, self.compname, self.format) f.close() def test_write_context_manager_calls_close(self): @@ -257,7 +263,7 @@ def test_read_params(self): f = self.f = self.module.open(self.sndfilepath) #self.assertEqual(f.getfp().name, self.sndfilepath) self.check_params(f, self.nchannels, self.sampwidth, self.framerate, - self.sndfilenframes, self.comptype, self.compname) + self.sndfilenframes, self.comptype, self.compname, self.format) def test_close(self): with open(self.sndfilepath, 'rb') as testfile: @@ -298,6 +304,8 @@ def test_read(self): f.setpos(f.getnframes() + 1) def test_copy(self): + if self.readonly: + self.skipTest('Read only file format') f = self.f = self.module.open(self.sndfilepath) fout = self.fout = self.module.open(TESTFN, 'wb') fout.setparams(f.getparams()) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 4526efe4b80ef4..e0c32976fd6f0d 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -6605,6 +6605,20 @@ def test_remainder(self): args = parser.parse_args(['--foo', 'a', '--', 'b', '--', 'c']) self.assertEqual(NS(foo='a', bar=['--', 'b', '--', 'c']), args) + def test_optional_remainder(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', nargs='...') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--', '--foo', 'a', 'b']) + self.assertEqual(NS(foo=None, bar=['--foo', 'a', 'b']), args) + args = parser.parse_args(['--foo', '--', 'a', 'b']) + self.assertEqual(NS(foo=['--', 'a', 'b'], bar=[]), args) + args = parser.parse_args(['--foo', 'a', '--', 'b']) + self.assertEqual(NS(foo=['a', '--', 'b'], bar=[]), args) + args = parser.parse_args(['--foo', 'a', 'b', '--']) + self.assertEqual(NS(foo=['a', 'b', '--'], bar=[]), args) + def test_subparser(self): parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('foo') diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index dda42cb33072c3..efa85b564f7cdf 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -565,7 +565,19 @@ def __repr__(self): g_partial = functools.partial(func, trigger, None, None, None, None, arg=None) self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), EvilObject, None, None, None, None, arg=None)") + def test_str_subclass_error(self): + class BadStr(str): + def __eq__(self, other): + raise RuntimeError + def __hash__(self): + return str.__hash__(self) + + def f(**kwargs): + return kwargs + p = functools.partial(f, poison="") + with self.assertRaises(RuntimeError): + result = p(**{BadStr("poison"): "new_value"}) @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 05e7d377517798..86886ac768f6d2 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,7 +1,7 @@ import unittest from test import audiotests from test import support -from test.support.os_helper import FakePath +from test.support.os_helper import FakePath, unlink import io import os import struct @@ -22,6 +22,7 @@ class WavePCM8Test(WaveTest, unittest.TestCase): sampwidth = 1 framerate = 11025 nframes = 48 + format = wave.WAVE_FORMAT_PCM comptype = 'NONE' compname = 'not compressed' frames = bytes.fromhex("""\ @@ -39,6 +40,7 @@ class WavePCM16Test(WaveTest, unittest.TestCase): sampwidth = 2 framerate = 11025 nframes = 48 + format = wave.WAVE_FORMAT_PCM comptype = 'NONE' compname = 'not compressed' frames = bytes.fromhex("""\ @@ -60,6 +62,7 @@ class WavePCM24Test(WaveTest, unittest.TestCase): sampwidth = 3 framerate = 11025 nframes = 48 + format = wave.WAVE_FORMAT_PCM comptype = 'NONE' compname = 'not compressed' frames = bytes.fromhex("""\ @@ -87,6 +90,8 @@ class WavePCM24ExtTest(WaveTest, unittest.TestCase): sampwidth = 3 framerate = 11025 nframes = 48 + format = wave.WAVE_FORMAT_EXTENSIBLE + readonly = True # Writing EXTENSIBLE wave format is not supported. comptype = 'NONE' compname = 'not compressed' frames = bytes.fromhex("""\ @@ -114,6 +119,7 @@ class WavePCM32Test(WaveTest, unittest.TestCase): sampwidth = 4 framerate = 11025 nframes = 48 + format = wave.WAVE_FORMAT_PCM comptype = 'NONE' compname = 'not compressed' frames = bytes.fromhex("""\ @@ -134,14 +140,142 @@ class WavePCM32Test(WaveTest, unittest.TestCase): frames = wave._byteswap(frames, 4) +class WaveIeeeFloatingPointTest(WaveTest, unittest.TestCase): + sndfilename = 'pluck-float32.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 4 + framerate = 11025 + nframes = 48 + format = wave.WAVE_FORMAT_IEEE_FLOAT + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 3C8B5960BA231400 3F16B41F3BFA5480 3EC44F0E3D1DC580 BF7E46533D843040 \ + BED084FC3D564C30 3F1153303CFCBE40 BF002FB73C583EC0 3CDAFEE0BC425180 \ + BF0F5154BD3826E0 BF169F56BDCAFD40 3EA660C0BE21A4EC 3E52E53CBE49332C \ + BE102E0CBE5B7214 BEE76852BE6C3BDC 3DE05A98BE7A4980 BE06B6B4BE7EB6EC \ + 3F2EB1B0BE6C7CC8 BD195500BE3E0F4C 3E1BBDF8BE03DFEC BE9F4E92BD8D8D58 \ + BF50E1D4BD111750 BDA079B0BCFBFB20 3D866358BD0C7640 BE833C0EBD17E240 \ + 3E0BFF04BD3978F0 3EFB9AE2BD14A780 BF0710B9BCD342E0 3F4DADB5BBA0CD80 \ + BEC3B11A3D024EB0 3F063AD33D97A8C0 BEF912803DEC74E0 3F2241733E1515D4 \ + BE0904D83E3AA604 BF7BF2003E3325BC 3F8000003E2229FC BF8000003E14A738 \ + 3F1338363DEB3B28 3F256E7C3DDBCA00 BE026A683DF5FD88 BEC70C923DFBE128 \ + BE5A5B183DCEA2D8 3F4689513DA5A7C8 3D8C8FE83DA1FFF0 3EAEE61C3DB0A0E0 \ + 3F2290DF3DE44E18 BF6867373E09D82C BF1216283DEEB360 3F08262F3DA5B488 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 4) + class MiscTestCase(unittest.TestCase): def test__all__(self): - not_exported = {'WAVE_FORMAT_PCM', 'WAVE_FORMAT_EXTENSIBLE', 'KSDATAFORMAT_SUBTYPE_PCM'} + not_exported = {'KSDATAFORMAT_SUBTYPE_PCM'} support.check__all__(self, wave, not_exported=not_exported) class WaveLowLevelTest(unittest.TestCase): + def test_setparams_6_tuple_defaults_to_pcm(self): + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + filename = fp.name + self.addCleanup(unlink, filename) + + with wave.open(filename, 'wb') as w: + w.setformat(wave.WAVE_FORMAT_IEEE_FLOAT) + w.setparams((1, 2, 22050, 0, 'NONE', 'not compressed')) + self.assertEqual(w.getformat(), wave.WAVE_FORMAT_PCM) + + def test_setparams_7_tuple_uses_format(self): + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + filename = fp.name + self.addCleanup(unlink, filename) + + with wave.open(filename, 'wb') as w: + w.setparams((1, 4, 22050, 0, 'NONE', 'not compressed', + wave.WAVE_FORMAT_IEEE_FLOAT)) + self.assertEqual(w.getformat(), wave.WAVE_FORMAT_IEEE_FLOAT) + + def test_setparams_7_tuple_ieee_64bit_sampwidth(self): + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + filename = fp.name + self.addCleanup(unlink, filename) + + with wave.open(filename, 'wb') as w: + w.setparams((1, 8, 22050, 0, 'NONE', 'not compressed', + wave.WAVE_FORMAT_IEEE_FLOAT)) + self.assertEqual(w.getformat(), wave.WAVE_FORMAT_IEEE_FLOAT) + self.assertEqual(w.getsampwidth(), 8) + + def test_getparams_backward_compatible_shape(self): + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + filename = fp.name + self.addCleanup(unlink, filename) + + with wave.open(filename, 'wb') as w: + w.setparams((1, 4, 22050, 0, 'NONE', 'not compressed', + wave.WAVE_FORMAT_IEEE_FLOAT)) + params = w.getparams() + self.assertEqual(params, (1, 4, 22050, 0, 'NONE', 'not compressed')) + + def test_getformat_setformat(self): + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + filename = fp.name + self.addCleanup(unlink, filename) + + with wave.open(filename, 'wb') as w: + w.setnchannels(1) + w.setsampwidth(4) + w.setframerate(22050) + self.assertEqual(w.getformat(), wave.WAVE_FORMAT_PCM) + w.setformat(wave.WAVE_FORMAT_IEEE_FLOAT) + self.assertEqual(w.getformat(), wave.WAVE_FORMAT_IEEE_FLOAT) + + def test_setformat_ieee_requires_32_or_64_bit_sampwidth(self): + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + filename = fp.name + self.addCleanup(unlink, filename) + + with wave.open(filename, 'wb') as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(22050) + with self.assertRaisesRegex(wave.Error, + 'unsupported sample width for IEEE float format'): + w.setformat(wave.WAVE_FORMAT_IEEE_FLOAT) + + def test_setsampwidth_ieee_requires_32_or_64_bit(self): + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + filename = fp.name + self.addCleanup(unlink, filename) + + with wave.open(filename, 'wb') as w: + w.setnchannels(1) + w.setframerate(22050) + w.setformat(wave.WAVE_FORMAT_IEEE_FLOAT) + with self.assertRaisesRegex(wave.Error, + 'unsupported sample width for IEEE float format'): + w.setsampwidth(2) + w.setsampwidth(4) + + def test_setsampwidth_ieee_accepts_64_bit(self): + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + filename = fp.name + self.addCleanup(unlink, filename) + + with wave.open(filename, 'wb') as w: + w.setnchannels(1) + w.setframerate(22050) + w.setformat(wave.WAVE_FORMAT_IEEE_FLOAT) + w.setsampwidth(8) + self.assertEqual(w.getsampwidth(), 8) + + def test_read_getformat(self): + b = b'RIFF' + struct.pack(' 4: + if self._format == WAVE_FORMAT_IEEE_FLOAT: + if sampwidth not in (4, 8): + raise Error('unsupported sample width for IEEE float format') + elif sampwidth < 1 or sampwidth > 4: raise Error('bad sample width') self._sampwidth = sampwidth @@ -519,6 +547,18 @@ def setcomptype(self, comptype, compname): self._comptype = comptype self._compname = compname + def setformat(self, format): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + if format not in (WAVE_FORMAT_IEEE_FLOAT, WAVE_FORMAT_PCM): + raise Error('unsupported wave format') + if format == WAVE_FORMAT_IEEE_FLOAT and self._sampwidth and self._sampwidth not in (4, 8): + raise Error('unsupported sample width for IEEE float format') + self._format = format + + def getformat(self): + return self._format + def getcomptype(self): return self._comptype @@ -526,10 +566,15 @@ def getcompname(self): return self._compname def setparams(self, params): - nchannels, sampwidth, framerate, nframes, comptype, compname = params if self._datawritten: raise Error('cannot change parameters after starting to write') + if len(params) == 6: + nchannels, sampwidth, framerate, nframes, comptype, compname = params + format = WAVE_FORMAT_PCM + else: + nchannels, sampwidth, framerate, nframes, comptype, compname, format = params self.setnchannels(nchannels) + self.setformat(format) self.setsampwidth(sampwidth) self.setframerate(framerate) self.setnframes(nframes) @@ -590,6 +635,9 @@ def _ensure_header_written(self, datasize): raise Error('sampling rate not specified') self._write_header(datasize) + def _needs_fact_chunk(self): + return self._format == WAVE_FORMAT_IEEE_FLOAT + def _write_header(self, initlength): assert not self._headerwritten self._file.write(b'RIFF') @@ -600,12 +648,23 @@ def _write_header(self, initlength): self._form_length_pos = self._file.tell() except (AttributeError, OSError): self._form_length_pos = None - self._file.write(struct.pack('kw, key)) { + int contains = PyDict_Contains(pto->kw, key); + if (contains < 0) { + goto error; + } + else if (contains == 1) { if (pto_kw_merged == NULL) { pto_kw_merged = PyDict_Copy(pto->kw); if (pto_kw_merged == NULL) { diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 948a413d98ddcd..12480e2cb1962f 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -141,10 +141,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } @@ -289,10 +290,10 @@ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - res = PyStackRef_FromPyObjectSteal(res_o); - if (PyStackRef_IsNull(res)) { + if (res_o == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; } @@ -521,10 +522,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } @@ -1270,10 +1272,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } diff --git a/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict b/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict index d36f55a40905d2..322d8180f7baa5 100644 --- a/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict +++ b/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict @@ -54,7 +54,7 @@ " " "\\t" ":\\n " -"\\\n" +"\\\\n" # type signatures and functions "-> " diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 39671d1ab51dfa..e2ab04cc2073c5 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -321,6 +321,9 @@ zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key) } PyObject *weak_cache = get_weak_cache(state, type); + if (weak_cache == NULL) { + return NULL; + } instance = PyObject_CallMethod(weak_cache, "get", "O", key, Py_None); if (instance == NULL) { Py_DECREF(weak_cache); @@ -505,6 +508,9 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, { zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); PyObject *weak_cache = get_weak_cache(state, type); + if (weak_cache == NULL) { + return NULL; + } if (only_keys == NULL || only_keys == Py_None) { PyObject *rv = PyObject_CallMethod(weak_cache, "clear", NULL); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2fb2c33428255a..e170fd65a20f64 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -710,10 +710,11 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { ERROR_NO_POP(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; INPUTS_DEAD(); @@ -729,10 +730,11 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { ERROR_NO_POP(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; INPUTS_DEAD(); @@ -748,10 +750,11 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { ERROR_NO_POP(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; INPUTS_DEAD(); @@ -772,10 +775,10 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - res = PyStackRef_FromPyObjectSteal(res_o); - if (PyStackRef_IsNull(res)) { + if (res_o == NULL) { ERROR_NO_POP(); } + res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; INPUTS_DEAD(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d89d512fd5630a..d33c67dd745bbf 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4620,11 +4620,12 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4656,14 +4657,15 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { stack_pointer[0] = right; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4696,8 +4698,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { stack_pointer[0] = left; stack_pointer[1] = right; stack_pointer += 2; @@ -4705,6 +4707,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4733,11 +4736,12 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4769,14 +4773,15 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { stack_pointer[0] = right; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4809,8 +4814,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { stack_pointer[0] = left; stack_pointer[1] = right; stack_pointer += 2; @@ -4818,6 +4823,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4846,11 +4852,12 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4882,14 +4889,15 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { stack_pointer[0] = right; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4922,8 +4930,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { stack_pointer[0] = left; stack_pointer[1] = right; stack_pointer += 2; @@ -4931,6 +4939,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; _tos_cache2 = r; @@ -4957,11 +4966,11 @@ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - res = PyStackRef_FromPyObjectSteal(res_o); - if (PyStackRef_IsNull(res)) { + if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; _tos_cache2 = r; @@ -4991,14 +5000,14 @@ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - res = PyStackRef_FromPyObjectSteal(res_o); - if (PyStackRef_IsNull(res)) { + if (res_o == NULL) { stack_pointer[0] = right; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; _tos_cache2 = r; @@ -5029,8 +5038,7 @@ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - res = PyStackRef_FromPyObjectSteal(res_o); - if (PyStackRef_IsNull(res)) { + if (res_o == NULL) { stack_pointer[0] = left; stack_pointer[1] = right; stack_pointer += 2; @@ -5038,6 +5046,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; _tos_cache2 = r; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a66d6ccff2d82d..628f0cc4d37310 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -141,10 +141,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } @@ -289,10 +290,10 @@ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - res = PyStackRef_FromPyObjectSteal(res_o); - if (PyStackRef_IsNull(res)) { + if (res_o == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; } @@ -521,10 +522,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } @@ -1270,10 +1272,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 83a11f613a2db7..f09bf778587b12 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1601,91 +1601,6 @@ uop_optimize( * Executor management ****************************************/ -/* We use a bloomfilter with k = 6, m = 256 - * The choice of k and the following constants - * could do with a more rigorous analysis, - * but here is a simple analysis: - * - * We want to keep the false positive rate low. - * For n = 5 (a trace depends on 5 objects), - * we expect 30 bits set, giving a false positive - * rate of (30/256)**6 == 2.5e-6 which is plenty - * good enough. - * - * However with n = 10 we expect 60 bits set (worst case), - * giving a false positive of (60/256)**6 == 0.0001 - * - * We choose k = 6, rather than a higher number as - * it means the false positive rate grows slower for high n. - * - * n = 5, k = 6 => fp = 2.6e-6 - * n = 5, k = 8 => fp = 3.5e-7 - * n = 10, k = 6 => fp = 1.6e-4 - * n = 10, k = 8 => fp = 0.9e-4 - * n = 15, k = 6 => fp = 0.18% - * n = 15, k = 8 => fp = 0.23% - * n = 20, k = 6 => fp = 1.1% - * n = 20, k = 8 => fp = 2.3% - * - * The above analysis assumes perfect hash functions, - * but those don't exist, so the real false positive - * rates may be worse. - */ - -#define K 6 - -#define SEED 20221211 - -/* TO DO -- Use more modern hash functions with better distribution of bits */ -static uint64_t -address_to_hash(void *ptr) { - assert(ptr != NULL); - uint64_t uhash = SEED; - uintptr_t addr = (uintptr_t)ptr; - for (int i = 0; i < SIZEOF_VOID_P; i++) { - uhash ^= addr & 255; - uhash *= (uint64_t)PyHASH_MULTIPLIER; - addr >>= 8; - } - return uhash; -} - -void -_Py_BloomFilter_Init(_PyBloomFilter *bloom) -{ - for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { - bloom->bits[i] = 0; - } -} - -/* We want K hash functions that each set 1 bit. - * A hash function that sets 1 bit in M bits can be trivially - * derived from a log2(M) bit hash function. - * So we extract 8 (log2(256)) bits at a time from - * the 64bit hash. */ -void -_Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) -{ - uint64_t hash = address_to_hash(ptr); - assert(K <= 8); - for (int i = 0; i < K; i++) { - uint8_t bits = hash & 255; - bloom->bits[bits >> 5] |= (1 << (bits&31)); - hash >>= 8; - } -} - -static bool -bloom_filter_may_contain(_PyBloomFilter *bloom, _PyBloomFilter *hashes) -{ - for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { - if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { - return false; - } - } - return true; -} - static int link_executor(_PyExecutorObject *executor, const _PyBloomFilter *bloom) {