From 3550898ca2deef29e2f94e7cb6dc59fbcf4dcd23 Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Fri, 18 Aug 2023 17:07:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84=E4=BB=A3=E7=90=86=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source_code/proxy.md | 4 ++-- feapder/VERSION | 2 +- feapder/network/proxy_pool/__init__.py | 4 ++-- feapder/network/proxy_pool/base.py | 2 +- .../proxy_pool/{memory_proxy_pool.py => proxy_pool.py} | 4 ++-- feapder/network/request.py | 4 ++-- feapder/setting.py | 2 +- feapder/templates/project_template/setting.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) rename feapder/network/proxy_pool/{memory_proxy_pool.py => proxy_pool.py} (95%) diff --git a/docs/source_code/proxy.md b/docs/source_code/proxy.md index 4863742e..de87845a 100644 --- a/docs/source_code/proxy.md +++ b/docs/source_code/proxy.md @@ -53,9 +53,9 @@ ip:port 1. 编写代理池:例如在你的项目下创建个my_proxypool.py,实现下面的函数 ```python - from feapder.network.proxy_pool import ProxyPool + from feapder.network.proxy_pool import BaseProxyPool - class MyProxyPool(ProxyPool): + class MyProxyPool(BaseProxyPool): def get_proxy(self): """ 获取代理 diff --git a/feapder/VERSION b/feapder/VERSION index d2c4b271..8b315b3f 100644 --- a/feapder/VERSION +++ b/feapder/VERSION @@ -1 +1 @@ -1.8.7 \ No newline at end of file +1.8.8 \ No newline at end of file diff --git a/feapder/network/proxy_pool/__init__.py b/feapder/network/proxy_pool/__init__.py index 7d8305cd..0a6935b6 100644 --- a/feapder/network/proxy_pool/__init__.py +++ b/feapder/network/proxy_pool/__init__.py @@ -7,5 +7,5 @@ @author: Boris @email: boris_liu@foxmail.com """ -from .base import ProxyPool -from .memory_proxy_pool import MemoryProxyPool +from .base import BaseProxyPool +from .proxy_pool import ProxyPool diff --git a/feapder/network/proxy_pool/base.py b/feapder/network/proxy_pool/base.py index ebef5744..0a2dc590 100644 --- a/feapder/network/proxy_pool/base.py +++ b/feapder/network/proxy_pool/base.py @@ -13,7 +13,7 @@ from feapder.utils.log import log -class ProxyPool: +class BaseProxyPool: @abc.abstractmethod def get_proxy(self): """ diff --git a/feapder/network/proxy_pool/memory_proxy_pool.py b/feapder/network/proxy_pool/proxy_pool.py similarity index 95% rename from feapder/network/proxy_pool/memory_proxy_pool.py rename to feapder/network/proxy_pool/proxy_pool.py index 2fc8f1ca..ce492633 100644 --- a/feapder/network/proxy_pool/memory_proxy_pool.py +++ b/feapder/network/proxy_pool/proxy_pool.py @@ -12,12 +12,12 @@ import requests import feapder.setting as setting -from feapder.network.proxy_pool.base import ProxyPool +from feapder.network.proxy_pool.base import BaseProxyPool from feapder.utils import metrics from feapder.utils import tools -class MemoryProxyPool(ProxyPool): +class ProxyPool(BaseProxyPool): """ 通过API提取代理,存储在内存中,无代理时会自动提取 API返回的代理以 \r\n 分隔 diff --git a/feapder/network/request.py b/feapder/network/request.py index 7a7f5637..b46b6558 100644 --- a/feapder/network/request.py +++ b/feapder/network/request.py @@ -21,7 +21,7 @@ from feapder.db.redisdb import RedisDB from feapder.network import user_agent from feapder.network.downloader.base import Downloader, RenderDownloader -from feapder.network.proxy_pool import ProxyPool +from feapder.network.proxy_pool import BaseProxyPool from feapder.network.response import Response from feapder.utils.log import log @@ -31,7 +31,7 @@ class Request: user_agent_pool = user_agent - proxies_pool: ProxyPool = None + proxies_pool: BaseProxyPool = None cache_db = None # redis / pika cached_redis_key = None # 缓存response的文件文件夹 response_cached:cached_redis_key:md5 diff --git a/feapder/setting.py b/feapder/setting.py index 1f9723d2..f8b7758f 100644 --- a/feapder/setting.py +++ b/feapder/setting.py @@ -133,7 +133,7 @@ PROXY_EXTRACT_API = None # 代理提取API ,返回的代理分割符为\r\n PROXY_ENABLE = True PROXY_MAX_FAILED_TIMES = 5 # 代理最大失败次数,超过则不使用,自动删除 -PROXY_POOL = "feapder.network.proxy_pool.MemoryProxyPool" # 代理池 +PROXY_POOL = "feapder.network.proxy_pool.ProxyPool" # 代理池 # 随机headers RANDOM_HEADERS = True diff --git a/feapder/templates/project_template/setting.py b/feapder/templates/project_template/setting.py index 2d11dc65..c18f2eee 100644 --- a/feapder/templates/project_template/setting.py +++ b/feapder/templates/project_template/setting.py @@ -122,7 +122,7 @@ # PROXY_EXTRACT_API = None # 代理提取API ,返回的代理分割符为\r\n # PROXY_ENABLE = True # PROXY_MAX_FAILED_TIMES = 5 # 代理最大失败次数,超过则不使用,自动删除 -# PROXY_POOL = "feapder.network.proxy_pool.MemoryProxyPool" # 代理池 +# PROXY_POOL = "feapder.network.proxy_pool.ProxyPool" # 代理池 # # # 随机headers # RANDOM_HEADERS = True