-
Notifications
You must be signed in to change notification settings - Fork 2
/
controller.py
118 lines (98 loc) · 3.91 KB
/
controller.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
'''
lupro 内部控制器
'''
from .typing import Any
from .api import generator, Batchsubmission, BulkDownload, Batchanalysis, xpath_Batchanalysis, json_Batchanalysis, re_Batchanalysis
from .publictool import reconfig, abnormal, original, persistence
import atexit
# 批量任务控制器
class batch():
'''批量任务控制器'''
def __init__(self, instantiation, url : list, filenameNo : list = []) -> list:
'''实例化生成器
Args:
`instantiation` : `lupro` lupro模板实例
`url` : `list` 链接表
`filenameNo` : `list` filename 序列且此序列会继承 `instantiation.filename`
Returns:
None
'''
self.name = instantiation.filename
self.generator = generator(instantiation, url, filenameNo)
self.filenameNo = [i.filename for i in self.generator]
persistence.shelve.add(self.name, {})
# 任务自省
@reconfig(config = persistence.ENABLED)
def province(self):
'''任务自省'''
fail, success, task = [], [], []
dbdict = persistence.shelve.put()
for i,j in enumerate(self.filenameNo):
if not j in dbdict:
fail.append(j)
task.append(self.generator[i])
success.append('-') # not perfect
else:
success.append(dbdict[j])
persistence.shelve.add(self.name, {'success' : success, 'task' : task, 'filenameNo' : self.filenameNo})
print(f"批量任务 {self.name} >> 一共有 {len(self.filenameNo)} 次请求", f"失败了 {len(fail)} 次" ,sep = '\n')
if not len(fail)==0:
res = '\n' + '\n'.join(fail)
print(f"分别是: {res}")
# 冷重启
@staticmethod
def coldheavy(filename) -> None:
'''冷重启
Args:
`filename` : `str` 冷重启对象的 `self.name`
Returns:
None
'''
Batchsubmission(persistence.shelve.put()[filename]['task'])
# 任务回调
@staticmethod
def callback(filename) -> list:
'''任务反持久化
Args:
`filename` : `str` 冷重启对象的 `self.name`
Returns:
list task列表
'''
container = persistence.shelve.put()
return {i:container.get(i) for i in container[filename]['filenameNo']}
# 批量请求
@abnormal
def Batchsubmission(self) -> list:
'''批量请求'''
atexit.register(batch.province, self)
return Batchsubmission(self.generator)
# 批量下载
@abnormal
def BulkDownload(self) -> list:
'''批量下载'''
atexit.register(batch.province, self)
return BulkDownload(self.generator)
# 批量解析
@abnormal
def Batchanalysis(self, mold : str , analytic : dict, auxiliary = original) -> list:
'''批量解析'''
atexit.register(batch.province, self)
return Batchanalysis(mold, self.generator, analytic, auxiliary)
# xpath 批量解析
@abnormal
def xpath_Batchanalysis(self, analytic, auxiliary = original) -> list:
'''xpath 批量解析'''
atexit.register(batch.province, self)
return xpath_Batchanalysis(self.generator, analytic, auxiliary)
# json 批量解析
@abnormal
def json_Batchanalysis(self, analytic, auxiliary = original) -> list:
'''json 批量解析'''
atexit.register(batch.province, self)
return json_Batchanalysis(self, analytic, auxiliary)
# 正则 批量解析
@abnormal
def re_Batchanalysis(self, analytic, auxiliary = original) -> list:
'''正则 批量解析'''
atexit.register(batch.province, self)
return re_Batchanalysis(self.generator, analytic, auxiliary)