-
Notifications
You must be signed in to change notification settings - Fork 0
/
retrieveurls.py
54 lines (47 loc) · 1.88 KB
/
retrieveurls.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import pandas as pd
import time
import csv
# PREPARE .CSV TO STORE PROFILE LINKS #
exportcsv = open('Links.csv', 'w', newline='')
csvwriter = csv.writer(exportcsv)
csvwriter.writerow(['Profile Link'])
# LOAD RESULT PAGE ON A PREVIOUSLY OPENED BROWSER WINDOW #
options = Options()
options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
chrome = r"""C:\Users\HP\Desktop\chromedriver.exe""" # ChromeDriver is needed, select here the folder where it is stored
driver = webdriver.Chrome(chrome, options=options)
load = driver.get('https://www.lovescout24.de/d/search/results')
driver.maximize_window()
time.sleep(3)
# SCROLL DOWN & DISPLAY MORE RESULTS #
driver.execute_script("window.scrollBy(0,document.body.scrollHeight)")
time.sleep(3)
driver.execute_script("window.scrollBy(0,document.body.scrollHeight)")
time.sleep(3)
seemore = driver.find_element_by_class_name('see-more-profiles-cta')
seemore.click()
time.sleep(3)
driver.execute_script("window.scrollBy(0,document.body.scrollHeight)")
time.sleep(3)
driver.execute_script("window.scrollBy(0,document.body.scrollHeight)")
time.sleep(3)
seemore = driver.find_element_by_class_name('see-more-profiles-cta')
seemore.click()
time.sleep(3)
driver.execute_script("window.scrollBy(0,document.body.scrollHeight)")
time.sleep(3)
driver.execute_script("window.scrollBy(0,document.body.scrollHeight)")
time.sleep(3)
# RETRIEVE PROFILE LINKS #
links = driver.find_elements_by_xpath("//a[contains(@href, '/d/profile-display/')]")
for ii in links:
print(ii.get_attribute('href'))
# EXPORT PROFILE LINKS TO .CSV #
csvwriter.writerow([ii.get_attribute('href')])
exportcsv.close()
# REMOVE DUPLICATE ENTRIES #
df = pd.read_csv('Links.csv')
df.drop_duplicates(subset=None, inplace=True)
df.to_csv('Links.csv', index=False)