From 867da5397425cddaf7423172a8d22f5ebb55c36f Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 1 Jul 2026 23:25:47 +1000 Subject: [PATCH] requests: Support unicode in json and data payloads. `json.dumps()` returns a str object, and if the JSON data contains unicode then computing the length of this str -- which is used for "Content-Length" -- will return the number of characters which will be shorter than the JSON payload that is sent as bytes. The server will then read a truncated body. Fix that by converting any str payload into bytes. This fix will also convert any str object passed in by the user as the `data` argument, and that actually seems to match the upstream requests behaviour (even if it's documented that `data` should be bytes). Note that `bytes(str, "utf-8")` does not make a copy of the str data, it just makes a small bytes object that wraps the same underlying data. So it's relatively efficient. Fixes issue #251. Signed-off-by: Damien George --- python-ecosys/requests/manifest.py | 2 +- python-ecosys/requests/requests/__init__.py | 7 ++- python-ecosys/requests/test_requests.py | 51 +++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index 924d12d72..4f76af397 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="1.0.1", pypi="requests") +metadata(version="1.0.2", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index 93aaa8afe..41f5cc47b 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -153,8 +153,11 @@ def request( if chunked_data: if "Transfer-Encoding" not in headers and "Content-Length" not in headers: headers["Transfer-Encoding"] = "chunked" - elif "Content-Length" not in headers: - headers["Content-Length"] = str(len(data)) + else: + if isinstance(data, str): + data = bytes(data, "utf-8") + if "Content-Length" not in headers: + headers["Content-Length"] = str(len(data)) if "Connection" not in headers: headers["Connection"] = "close" diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index 3c8683674..7b7a8f6c3 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -95,6 +95,53 @@ def test_post_json(): ), format_message(response) +def test_post_json_unicode(): + response = requests.request("POST", "http://example.com", json="aαbβcγdδ") # noqa: RUF001 + + assert response.raw._write_buffer.getvalue() == ( + b"POST / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Content-Type: application/json\r\n" + b"Host: example.com\r\n" + b"Content-Length: 14\r\n\r\n" + bytes('"aαbβcγdδ"', "utf-8") # noqa: RUF001 + ), format_message(response) + + +def test_post_data_str(): + response = requests.request("POST", "http://example.com", data="body") + + assert response.raw._write_buffer.getvalue() == ( + b"POST / HTTP/1.1\r\n" + b"Content-Length: 4\r\n" + b"Host: example.com\r\n" + b"Connection: close\r\n\r\n" + b"body" + ), format_message(response) + + +def test_post_data_str_unicode(): + response = requests.request("POST", "http://example.com", data="aαbβcγdδ") # noqa: RUF001 + + assert response.raw._write_buffer.getvalue() == ( + b"POST / HTTP/1.1\r\n" + b"Content-Length: 12\r\n" + b"Host: example.com\r\n" + b"Connection: close\r\n\r\n" + bytes("aαbβcγdδ", "utf-8") # noqa: RUF001 + ), format_message(response) + + +def test_post_data_bytes(): + response = requests.request("POST", "http://example.com", data=b"body\x01\x02\x03\xff") + + assert response.raw._write_buffer.getvalue() == ( + b"POST / HTTP/1.1\r\n" + b"Content-Length: 8\r\n" + b"Host: example.com\r\n" + b"Connection: close\r\n\r\n" + b"body\x01\x02\x03\xff" + ), response.raw._write_buffer.getvalue() + + def test_post_chunked_data(): def chunks(): yield "test" @@ -268,6 +315,10 @@ def test_redirect_relative(): test_get_auth() test_get_custom_header() test_post_json() +test_post_json_unicode() +test_post_data_str() +test_post_data_str_unicode() +test_post_data_bytes() test_post_chunked_data() test_overwrite_get_headers() test_overwrite_post_json_headers()