-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcian.py
207 lines (177 loc) · 8.59 KB
/
cian.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
import requests
import re
import urllib
from bs4 import BeautifulSoup
import time
import threading
import csv
import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import file_treatment
start_time = time.time()
#https://kazan.cian.ru/sale/flat/235438508/ error
class House:
#driver = webdriver.Chrome(executable_path=r"C:\Users\731ru\Downloads\chromedriver\chromedriver.exe")
def __init__(self, link):
self.link = link
self.get_atribute()
def get_atribute(self):
res = urllib.request.urlopen(self.link).read().decode("utf-8")
self.check_captcha(res)
self.soup = BeautifulSoup(res, 'html.parser')
self.address = self.soup.find(class_="a10a3f92e9--geo--18qoo").span.get("content")#вытаскиваем адрес
info = self.soup.find_all(class_="a10a3f92e9--info-value--18c8R")#находим информацию про этаж
self.floor, self.max_floor = self.get_floor("(\d{1,10}) из (\d{1,10})", info)# ищем этаж
self.square = int(float(self.checking(info[0], "text","")[:-3].replace(",", ".")))#Находим общую площадь*
self.room = re.search("(\d{1,10})-комн.",self.soup.find(class_="a10a3f92e9--title--2Widg").text).group(1)#Кол-во комнат
self.repairs = self.checking(self.soup.find(class_="a10a3f92e9--value--3Ftu5"),"text", "")#ремонт
self.price = int(self.checking(self.soup.find(itemprop="price"), "", "content")[:-1].replace(" ", ""))#Стоимость*
self.one_price = self.checking(self.soup.find(class_="a10a3f92e9--price_per_meter--hKPtN a10a3f92e9--price_per_meter--residential--1mFDW"),"text", "")[:-4]#Цена за кв. метр
#self.ten_watches, self.data_create = self.get_watch()#за 10 дней и дата создания
self.all_watches = re.search("(\d{1,10}) просмо", self.soup.find(class_="a10a3f92e9--link--1t8n1 a10a3f92e9--link--2mJJk").text).group(1) # всего просмотров
self.type = self.checking(self.soup.find(class_="a10a3f92e9--vas_item--2wQfE"), "text", "")#тип размещения
self.first_price = self.get_first_price()#первоначальная цена*
self.district = self.get_district()#получение р-н
self.underground = self.get_underground()#метро
self.type_of_house = self.checking(self.soup.find(class_="a10a3f92e9--value--3Ftu5"),"text", "")#тип дома
self.result = [self.link, self.address, self.floor, self.max_floor, self.square, self.room, self.repairs, self.price,
self.one_price, self.all_watches, self.type,self.first_price, self.district, self.underground,
self.type_of_house] #self.ten_watches, self.data_create]
#print(self.result)
def check_captcha(self,res):
if 'id="form_captcha"' in res:
print("\nCAPTCHA\n")
exit(0)
def write_file(self, number, name):
first_line = ["Ссылка","Адрес", "Этаж", "Всего этажей", "Площадь","Кол-во комнат","Ремонт","Стоимость","Цена за кв. м.","Всего Просмотров",
"Тип объявления","Первоначальная цена","Район","Метро","Тип дома","Просмотры за 10 дней", "Дата создания"]
if number == 0:
write = "w"
else:
write = "a"
with open(name, write, newline="") as file:
writer = csv.writer(file)
if write == "w":
writer.writerow(first_line)
writer.writerow(self.result)
def checking(self, result, text, get): # Проверка что там находится не None
if result == None:
return "Не указано"
elif text:
return result.text
elif get:
return result.get(get)
return "Не указано"
def get_watch(self):
self.driver.get(self.link)
elem = self.driver.find_element_by_class_name("a10a3f92e9--link--1t8n1")
elem.click()
time.sleep(1)
place = self.driver.find_element_by_class_name("a10a3f92e9--information--AyP9e").text
return (re.search("(\d{1,10}) .* за последние 10 дней",place).group(1), re.search("\d{1,2}.\d{1,2}.\d{1,2}",place).group(0))#за 10 дней и дата создания
def get_first_price(self):
div = self.soup.find(class_="a10a3f92e9--container--3jr-Q")
if div == None:
return "Не указано"
else:
return div.find_all(class_="a10a3f92e9--event-price--BxH3c")[-1].text[:-1]
def get_district(self):#получение района
address = self.soup.find_all(class_="a10a3f92e9--link--1t8n1 a10a3f92e9--address-item--1clHr")
if address == None:
return "Не указано"
for i in range(len(address)):
if "р-н" in address[i].text:
return address[i].text
def get_underground(self):
result = ""
unders = self.soup.find_all(class_="a10a3f92e9--underground_link--AzxRC")
if unders != None:
for i in range(len(unders)):
result += unders[i].text + ","
return result
def get_floor(self, pattern, info):
for i in range(len(info)):
result = re.search(pattern, info[i].text)
if result != None:
return (int(result.group(1)),int(result.group(2)))
return "Не указано"
def close(self):
self.driver.close()
def read_file(name):
with open(name, 'br') as file:
return str(file.read(),encoding="utf-8")
url = "https://kazan.cian.ru/cat.php?deal_type=sale&engine_version=2&foot_min=45&metro%5B0%5D=319&offer_type=flat&only_foot=2®ion=4777&room3=1&room4=1"
url = "https://kazan.cian.ru/cat.php?deal_type=sale&engine_version=2&foot_min=45&metro%5B0%5D=316&metro%5B1%5D=317&metro%5B2%5D=318&metro%5B3%5D=319&metro%5B4%5D=320&metro%5B5%5D=321&object_type%5B0%5D=2&offer_type=flat&only_foot=2&room1=1&room2=1"
def get_link(soup):#получает список ссылок
list_of_flat = soup.find_all(class_="c6e8ba5398--header--1fV2A")
links = [_ for _ in range(len(list_of_flat))]
for i in range(len(list_of_flat)):
links[i] = list_of_flat[i].get('href')
print(links)
return links
def get_flats(url):
pages_number_list = []
i = 1
flats = []
while 1:
if i==2:
url = url + "&p=" + str(i)
else:
url = url.replace("&p="+str(i-1),"&p="+str(i))
res = urllib.request.urlopen(url).read().decode("utf-8")
soup = BeautifulSoup(res, 'html.parser')
pages_count = int(soup.find(class_="_93444fe79c--list-item--active--3dOSi").span.text)
if i == 1:
flats.append(get_link(soup))
i = i + 1
continue
if pages_count == 1: #or pages_count == 3:
print("now we break i - ", pages_count)
break
print(pages_count)
flats.append(get_link(soup))
i += 1
return flats
links = get_flats(url)
print("Найдено объявлений ", (len(links)-1)*len(links[0])+len(links[-1]))
medium = time.time()
print("\n", medium - start_time, "\n")
objects = []
def get_class(flats, start, step):
k = 0
for i in range(len(flats)):
if i == len(flats)/2:
print("\nMedium\n")
for j in range(start, len(flats[i]), step):
try:
k += 1
#print(k, start)
house = House(flats[i][j])
objects.append(house)
#if i==len(flats)-1 and j==len(flats[i])-1:
#house.close()
except Exception as e:
print("\n\n It was error: "+house.link + str(e) + " \n\n" )
continue
#get_class(links,0,1)#links, start point, end point
# 56 advert - 27 sec
# 56 advert - 37 sec
# 56 advert - 61 sec
th1 = threading.Thread(target=get_class, args=(links, 0, 3))
th2 = threading.Thread(target=get_class, args=(links, 1, 3))
th3 = threading.Thread(target=get_class, args=(links, 2, 3))
th1.start()
th2.start()
th3.start()
th1.join()
th2.join()
th3.join()
print("\n", time.time() - medium)
objects = sorted(objects, key=lambda object: (object.floor, object.square, object.price))
filename = "result_sort.csv"
for i in range(len(objects)):
objects[i].write_file(i, filename)
file_treatment.change_csv_to_xlsx(filename)
file_treatment.draw(filename)
print("\n", time.time() - start_time)