diff --git a/python-ecosys/requests/README.md b/python-ecosys/requests/README.md index d6ceaadc5..7f97aa103 100644 --- a/python-ecosys/requests/README.md +++ b/python-ecosys/requests/README.md @@ -4,7 +4,8 @@ This module provides a lightweight version of the Python [requests](https://requests.readthedocs.io/en/latest/) library. It includes support for all HTTP verbs, https, json decoding of responses, -redirects, basic authentication. +redirects, basic authentication, HTTP/1.1 requests, and reading response +bodies with Content-Length via streaming ``.raw`` or lazy ``.content``. ### Limitations @@ -14,3 +15,10 @@ redirects, basic authentication. * Compressed requests/responses are not currently supported. * File upload is not supported. * Chunked encoding in responses is not supported. +* HTTP keep-alive connection reuse is not supported (Connection: close by default). + +### Follow-up work + +* Chunked response bodies (separate PR). +* TLS certificate verification (see micropython-lib issue #838). +* ``stream=True`` incremental body reads (see issue #777). diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index 85f159753..caabb93cb 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.10.2", pypi="requests") +metadata(version="0.10.3", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index 68b4b18cb..4ef5b13b3 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -1,6 +1,26 @@ import socket +class BodyStream: + def __init__(self, sock, remaining): + self._sock = sock + self._remaining = remaining + + def read(self, n=-1): + if self._remaining == 0: + return b"" + if n < 0 or n > self._remaining: + n = self._remaining + data = self._sock.read(n) + self._remaining -= len(data) + if not data: + raise ValueError("Connection closed before Content-Length satisfied") + return data + + def close(self): + self._sock.close() + + class Response: def __init__(self, f): self.raw = f @@ -98,7 +118,7 @@ def request( context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) context.verify_mode = tls.CERT_NONE s = context.wrap_socket(s, server_hostname=host) - s.write(b"%s /%s HTTP/1.0\r\n" % (method, path)) + s.write(b"%s /%s HTTP/1.1\r\n" % (method, path)) if "Host" not in headers: headers["Host"] = host @@ -155,6 +175,7 @@ def request( reason = "" if len(l) > 2: reason = l[2].rstrip() + remaining = None while True: l = s.readline() if not l or l == b"\r\n": @@ -173,7 +194,10 @@ def request( elif parse_headers is True: l = str(l, "utf-8") k, v = l.split(":", 1) - resp_d[k] = v.strip() + v = v.strip() + resp_d[k] = v + if k.lower() == "content-length": + remaining = int(v) else: parse_headers(l, resp_d) except OSError: @@ -189,7 +213,10 @@ def request( else: return request(method, redirect, data, json, headers, stream) else: - resp = Response(s) + if remaining is not None: + resp = Response(BodyStream(s, remaining)) + else: + resp = Response(s) resp.status_code = status resp.reason = reason if resp_d is not None: diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index ac77291b0..9a5e55392 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -3,9 +3,9 @@ class Socket: - def __init__(self): + def __init__(self, read_data=b"HTTP/1.1 200 OK\r\n\r\n"): self._write_buffer = io.BytesIO() - self._read_buffer = io.BytesIO(b"HTTP/1.0 200 OK\r\n\r\n") + self._read_buffer = io.BytesIO(read_data) def connect(self, address): pass @@ -16,6 +16,12 @@ def write(self, buf): def readline(self): return self._read_buffer.readline() + def read(self, size=-1): + return self._read_buffer.read(size) + + def close(self): + pass + class socket: AF_INET = 2 @@ -30,135 +36,219 @@ def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP): return Socket() -sys.modules["socket"] = socket +def install_mock_socket(): + sys.modules["socket"] = socket + + +install_mock_socket() # ruff: noqa: E402 import requests -def format_message(response): - return response.raw._write_buffer.getvalue().decode("utf8") +def request_bytes(response): + raw = response.raw + if hasattr(raw, "_sock"): + raw = raw._sock + return raw._write_buffer.getvalue() + + +def assert_has(data, *parts): + for part in parts: + if part not in data: + raise AssertionError("missing {!r} in:\n{}".format(part, data)) def test_simple_get(): + install_mock_socket() response = requests.request("GET", "http://example.com") - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" + b"Connection: close\r\n" + b"Host: example.com\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"Host: example.com\r\n", + b"Connection: close\r\n", + b"\r\n", + ) + assert response.content == b"" def test_get_auth(): + install_mock_socket() response = requests.request( "GET", "http://example.com", auth=("test-username", "test-password") ) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Host: example.com\r\n" - + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n" - + b"Connection: close\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n", + b"Host: example.com\r\n", + b"Connection: close\r\n", + ) def test_get_custom_header(): + install_mock_socket() response = requests.request("GET", "http://example.com", headers={"User-Agent": "test-agent"}) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"User-Agent: test-agent\r\n" - + b"Host: example.com\r\n" - + b"Connection: close\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"User-Agent: test-agent\r\n", + b"Host: example.com\r\n", + b"Connection: close\r\n", + ) def test_post_json(): + install_mock_socket() response = requests.request("GET", "http://example.com", json="test") - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Connection: close\r\n" - + b"Content-Type: application/json\r\n" - + b"Host: example.com\r\n" - + b"Content-Length: 6\r\n\r\n" - + b'"test"' - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"Content-Type: application/json\r\n", + b"Host: example.com\r\n", + b"Content-Length: 6\r\n", + b"Connection: close\r\n", + b'"test"', + ) def test_post_chunked_data(): def chunks(): yield "test" + install_mock_socket() response = requests.request("GET", "http://example.com", data=chunks()) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Transfer-Encoding: chunked\r\n" - + b"Host: example.com\r\n" - + b"Connection: close\r\n\r\n" - + b"4\r\ntest\r\n" - + b"0\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"GET / HTTP/1.1\r\n", + b"Transfer-Encoding: chunked\r\n", + b"Host: example.com\r\n", + b"Connection: close\r\n", + b"4\r\ntest\r\n", + b"0\r\n\r\n", + ) def test_overwrite_get_headers(): + install_mock_socket() response = requests.request( "GET", "http://example.com", headers={"Host": "test.com", "Connection": "keep-alive"} ) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" + b"Connection: keep-alive\r\n" + b"Host: test.com\r\n\r\n" - ), format_message(response) + data = request_bytes(response) + assert_has(data, b"GET / HTTP/1.1\r\n", b"Host: test.com\r\n", b"Connection: keep-alive\r\n") def test_overwrite_post_json_headers(): + install_mock_socket() response = requests.request( "GET", "http://example.com", json="test", headers={"Content-Type": "text/plain", "Content-Length": "10"}, ) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Connection: close\r\n" - + b"Content-Length: 10\r\n" - + b"Content-Type: text/plain\r\n" - + b"Host: example.com\r\n\r\n" - + b'"test"' - ), format_message(response) + data = request_bytes(response) + assert_has( + data, + b"Content-Type: text/plain\r\n", + b"Content-Length: 10\r\n", + b"Host: example.com\r\n", + b'"test"', + ) def test_overwrite_post_chunked_data_headers(): def chunks(): yield "test" + install_mock_socket() response = requests.request( "GET", "http://example.com", data=chunks(), headers={"Content-Length": "4"} ) - - assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" - + b"Host: example.com\r\n" - + b"Content-Length: 4\r\n" - + b"Connection: close\r\n\r\n" - + b"test" - ), format_message(response) + data = request_bytes(response) + assert_has(data, b"Content-Length: 4\r\n", b"Host: example.com\r\n", b"test") def test_do_not_modify_headers_argument(): - global do_not_modify_this_dict - do_not_modify_this_dict = {} - requests.request("GET", "http://example.com", headers=do_not_modify_this_dict) - - assert do_not_modify_this_dict == {}, do_not_modify_this_dict - - -test_simple_get() -test_get_auth() -test_get_custom_header() -test_post_json() -test_post_chunked_data() -test_overwrite_get_headers() -test_overwrite_post_json_headers() -test_overwrite_post_chunked_data_headers() -test_do_not_modify_headers_argument() + install_mock_socket() + headers_arg = {} + requests.request("GET", "http://example.com", headers=headers_arg) + assert headers_arg == {}, headers_arg + + +def test_content_length_via_content(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello" + ) + response = requests.request("GET", "http://example.com") + assert response.content == b"hello" + assert response.headers["Content-Length"] == "5" + install_mock_socket() + + +def test_chunked_response_raises(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n" + ) + raised = False + try: + requests.request("GET", "http://example.com") + except ValueError as e: + raised = True + if "Unsupported" not in str(e): + raise + if not raised: + raise AssertionError("expected ValueError for chunked response") + install_mock_socket() + + +def test_raw_open_before_content(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello" + ) + response = requests.request("GET", "http://example.com") + assert response.raw is not None + assert response.raw.read(1) == b"h" + install_mock_socket() + + +def test_raw_incremental_content_length(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nabcdefghij" + ) + response = requests.request("GET", "http://example.com") + assert response.raw.read(3) == b"abc" + assert response.raw.read(3) == b"def" + assert response.content == b"ghij" + assert response.raw is None + install_mock_socket() + + +TESTS = ( + test_simple_get, + test_get_auth, + test_get_custom_header, + test_post_json, + test_post_chunked_data, + test_overwrite_get_headers, + test_overwrite_post_json_headers, + test_overwrite_post_chunked_data_headers, + test_do_not_modify_headers_argument, + test_content_length_via_content, + test_chunked_response_raises, + test_raw_open_before_content, + test_raw_incremental_content_length, +) + + +def run_all(): + for test in TESTS: + test() + + +if __name__ == "__main__": + run_all() + print("test_requests: {} tests OK".format(len(TESTS)))