Use coroutine to do cdn test.

README.md

@@ -1,7 +1,7 @@
 ### 12306 购票小助手
 #### python版本
   - [ ] 2.7.10 - 2.7.15
-  - [x] 3.6 - 3.7.4
+  - [x] 3.7 - 3.7.4
   - [ ] 2.7.9
 
 #### 已有功能

TickerConfig.py

@@ -154,4 +154,4 @@
 MIN_TIME = 1
 
 # 软件版本
-RE_VERSION = "1.2.004"
+RE_VERSION = "1.2.005"

Update.md

@@ -188,4 +188,7 @@
 
 -  2019.09.18更新
     - 修改下单问题
-    - 优化车次打印
+    - 优化车次打印
+
+-  2019.12.30更新
+    - 使用python3.7协程加速cdn测试过程

agency/cdn_utils.py

@@ -1,38 +1,38 @@
 # encoding=utf8
-import datetime
 import operator
 import os
-import requests
-from config import urlConf
-import threading
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
 from config.urlConf import urls
 
 from myUrllib.httpUtils import HTTPClient
 
-cdn_list = []
+finish_count = 0
 
 
-class CDNProxy(threading.Thread):
-    def __init__(self, cdns):
-        super().__init__()
-        self.cdns = cdns
-        self.urlConf = urlConf.urls
-        self.httpClint = requests
-        self.city_list = []
-        self.timeout = 5
+def calculate_rtt_blocking(cdn):
+    global finish_count
+    http = HTTPClient(0)
+    url = urls["loginInitCdn"]
+    http._cdn = cdn
+    rtt_dict = {"rtt": 9999}
+    http.send(url, elapsed=rtt_dict)
+    rtt = rtt_dict["rtt"]
+    finish_count += 1
+    # 过滤逻辑为当前cdn响应值小于1000毫秒
+    if rtt < 2000:
+        print(f"ip:{cdn}，延迟:{rtt}，已测试个数: {finish_count}")
+        return {"ip": cdn, "time": rtt}
+    else:
+        print(f"ip:{cdn}，延迟:N/A，已测试个数: {finish_count}")
+        return None
 
-    def run(self):
-        for cdn in self.cdns:
-            http = HTTPClient(0)
-            url = urls["loginInitCdn"]
-            http._cdn = cdn.replace("\n", "")
-            start_time = datetime.datetime.now()
-            rep = http.send(url)
-            retTime = (datetime.datetime.now() - start_time).microseconds / 1000
-            if rep and "message" not in rep and retTime < 3000:
-                if cdn.replace("\n", "") not in cdn_list:  # 如果有重复的cdn，则放弃加入
-                    print(f"加入cdn: {cdn}")
-                    cdn_list.append({"ip": cdn.replace("\n", ""), "time": retTime})
+
+async def calculate_all_rtt_async(cdns, loop, executor):
+    done, _ = await asyncio.wait(fs=[loop.run_in_executor(executor, calculate_rtt_blocking, cdn) for cdn in cdns],
+                                       return_when=asyncio.ALL_COMPLETED)
+    cdn_list = [task.result() for task in done if task.exception() is None and task.result() is not None]
+    return cdn_list
 
 
 def open_cdn_file(cdnFile):
@@ -52,7 +52,7 @@ def open_cdn_file(cdnFile):
             return cdn
 
 
-def sortCdn():
+def sortCdn(cdn_list):
     """
     对cdn进行排序
     :return:
@@ -73,20 +73,11 @@ def filterCdn():
     :return:
     """
     cdns = open_cdn_file("cdn_list")
-    cdnss = [cdns[i:i + 50] for i in range(0, len(cdns), 50)]
-    cdnThread = []
-    for cdn in cdnss:
-        t = CDNProxy(cdn)
-        cdnThread.append(t)
-    for cdn_t in cdnThread:
-        cdn_t.start()
-
-    for cdn_j in cdnThread:
-        cdn_j.join()
-
+    loop = asyncio.get_event_loop()
+    cdn_list = loop.run_until_complete(calculate_all_rtt_async(cdns, loop, ThreadPoolExecutor()))
     print(f"当前有效cdn个数为: {len(cdn_list)}")
     if cdn_list:
-        ips = sortCdn()
+        ips = sortCdn(cdn_list)
         path = os.path.join(os.path.dirname(__file__), f'../filter_cdn_list')
         f = open(path, "a+")
         f.seek(0)

docker-compose.yml

@@ -6,7 +6,7 @@ services:
         build:
             context: .
             dockerfile: ./Dockerfile37
-        image: ticket:v1.2.004
+        image: ticket:v1.2.005
         environment:
             - PYTHONUNBUFFERED=1
             - CAPTCHALOCAL=1

init/select_ticket_info.py

@@ -8,7 +8,7 @@
 import time
 import TickerConfig
 import wrapcache
-from agency.cdn_utils import CDNProxy, open_cdn_file
+from agency.cdn_utils import open_cdn_file
 from config import urlConf, configCommon
 from config.TicketEnmu import ticket
 from config.configCommon import seat_conf_2, seat_conf
@@ -60,8 +60,8 @@ def get_ticket_info():
         print(u"*" * 50)
         print(f"检查当前版本为: {TickerConfig.RE_VERSION}")
         version = sys.version.split(" ")[0]
-        print(u"检查当前python版本为：{}，目前版本只支持3.6以上".format(version))
-        if version < "3.6.0":
+        print(u"检查当前python版本为：{}，目前版本只支持3.7以上".format(version))
+        if version < "3.7.0":
             raise Exception
         print(u"12306刷票小助手，最后更新于2019.09.18，请勿作为商业用途，交流群号："
               u" 1群：286271084(已满)\n"

myUrllib/httpUtils.py

@@ -123,7 +123,7 @@ def cdn(self):
     def cdn(self, cdn):
         self._cdn = cdn
 
-    def send(self, urls, data=None, **kwargs):
+    def send(self, urls, data=None, elapsed=None, **kwargs):
         """send request to url.If response 200,return response, else return None."""
         allow_redirects = False
         is_logger = urls.get("is_logger", False)
@@ -173,6 +173,8 @@ def send(self, urls, data=None, **kwargs):
                                            allow_redirects=allow_redirects,
                                            verify=False,
                                            **kwargs)
+                if is_test_cdn:
+                    elapsed["rtt"] = round(response.elapsed.total_seconds() * 1000, 3)
                 if response.status_code == 200 or response.status_code == 302:
                     if urls.get("not_decode", False):
                         return response.content