feat: make the tester anonymity-check URL configurable via TEST_ANONYMOUS_URL (#234)

- setting: add TEST_ANONYMOUS_URL (default https://httpbin.org/ip)

- tester: use TEST_ANONYMOUS_URL instead of the hardcoded httpbin.org endpoint so users can self-host httpbin and avoid public rate limits

- README: document TEST_ANONYMOUS_URL and TEST_ANONYMOUS

Closes #109

Author: Germey

README.md

@@ -277,6 +277,8 @@ GET http://localhost:5555/all?area=CN
 - TEST_TIMEOUT：测试超时时间，默认 10 秒
 - TEST_BATCH：批量测试数量，默认 20 个代理
 - TEST_VALID_STATUS：测试有效的状态码
+- TEST_ANONYMOUS：是否只保留匿名代理，默认 true
+- TEST_ANONYMOUS_URL：匿名 / 出口 IP 检测地址，默认 `https://httpbin.org/ip`，需返回 httpbin 格式的 JSON（`{"origin": "1.2.3.4"}`）。可指向自建 httpbin 服务以避免公共服务限流
 - API_HOST：代理 Server 运行 Host，默认 0.0.0.0
 - API_PORT：代理 Server 运行端口，默认 5555
 - API_THREADED：代理 Server 是否使用多线程，默认 true

proxypool/processors/tester.py

@@ -4,7 +4,7 @@
 from proxypool.schemas import Proxy
 from proxypool.storages.redis import RedisClient
 from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \
-    TEST_DONT_SET_MAX_SCORE
+    TEST_DONT_SET_MAX_SCORE, TEST_ANONYMOUS_URL
 from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError, \
     ClientResponseError, ContentTypeError
 from asyncio import TimeoutError
@@ -49,7 +49,7 @@ async def test(self, proxy: Proxy, session: aiohttp.ClientSession):
             # the proxy has the effect of hiding the real IP
             # logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}')
             if TEST_ANONYMOUS:
-                url = 'https://httpbin.org/ip'
+                url = TEST_ANONYMOUS_URL
                 async with session.get(url, timeout=TEST_TIMEOUT) as response:
                     resp_json = await response.json()
                     origin_ip = resp_json['origin']

proxypool/setting.py

@@ -75,6 +75,10 @@
 TEST_BATCH = env.int('TEST_BATCH', 20)
 # only save anonymous proxy
 TEST_ANONYMOUS = env.bool('TEST_ANONYMOUS', True)
+# the url used to check the proxy anonymity and its exit ip;
+# must return json like httpbin.org/ip ({"origin": "1.2.3.4"});
+# point this to a self-hosted httpbin to avoid public rate limits
+TEST_ANONYMOUS_URL = env.str('TEST_ANONYMOUS_URL', 'https://httpbin.org/ip')
 # TEST_HEADERS = env.json('TEST_HEADERS', {
 #     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
 # })