-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathconfig.sample.toml
124 lines (107 loc) · 4.38 KB
/
config.sample.toml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# this is a sample configuration file
# the program will pick it up, if it is named config.toml
# if this is a testrun
# during a testrun no sink will be triggered
test = false
# number of threads on which the watchers will be executed
thread_count = 4
# if true, the first run will be used to setup
# only on the second run observers will be informed
initial_run = false
# whether properwatcher will continue to run after first loop
# if true, properwatcher will wait for <interval> seconds before it will trigger the next run
run_periodically = true
# set how often the scraping process will be triggered
# run every 5 minutes (300 seconds)
interval = 300
# supported filters:
# - dynamodb: filters out properties already in database
filters = [ ]
# supported enrichers:
# - nominatim: adds geocoordinates
enrichers = [ ]
# supported observers:
# - firebase: saves properties to a firebase collection
# - csv: appends found properties to a CSV file
# - mail: sends found properties via mail
# - telegram: sends found properties to a telegram chat
# - dynamodb: saves properties to a dynamodb table
observers = [ ]
#-------------
# watchers
#-------------
# there can be multiple watcher blocks
# a watcher defines an endpoint that will be scraped in the specified time interval
# properties found on any of the watched endpoints will be merged into one big result set
[[watcher]]
# this will be added as metadata to each property that is seen by this watcher
city = "Munich"
# this is the endpoint that you want to scrape
# make sure it contains your search criteria
address = "https://www.immobilienscout24.de/Suche/de/bayern/muenchen-kreis/wohnung-mieten?numberofrooms=2.0-&price=-1500.0&livingspace=40.0-&enteredFrom=one_step_search"
# available crawlers are: immoscout, immowelt, wohnungsboerse, wggesucht, sueddeutsche
crawler = "immoscout"
# available contract_types: buy, rent
contract_type = "rent"
# available property_types: house, flat
property_type = "flat"
#-------------
# modules
#-------------
# found properties can be geocoded (address will be translated into lat/long coordinates).
# this will decrease notification speed, because properwatcher will pause for 1 second
# after each request (see usage policy of nominatim)
[nominatim]
# nominatim API will be used for geocoding
# either use the global one, or you can also use a local instance
nominatim_url = "https://nominatim.openstreetmap.org/search"
# nominatim is a public free API used by openstreetmap
# we provide a custom user agent here, so they can rate limit their usage
# and block applications that exhaustively access their endpoints
# feel free to append a custom string i.e. properwatcher-flo
user_agent = "properwatcher"
# found properties can be sent to a given Telegram channel
# therefore, you first need to create a Telegram bot and start a conversation with it.
# you can follow this tutorial's first two steps to create a bot token and chat id:
# https://www.shellhacks.com/telegram-api-send-message-personal-notification-bot/
[telegram]
# API key of your Telegram bot
api_key = "<telegram-bot-api-key>"
# chat id of the conversation with your private bot
# can also be the id of a private group chat
chat_id = "<chat-id>"
# found properties can be forwarded to you via email
[mail]
smtp_server = "smtp.gmail.com"
# your username - in case of gmail, your mail address
# this is also where the notification mails are being sent to
username = "user"
# if you're going with gmail, you need to create an application specific password
# see here: https://support.google.com/mail/answer/185833?hl=en
password = "pass"
# found properties can be appended to a CSV file
# this option is recommended for easy dataset inspection via:
# tableau or microsoft's power bi
[csv]
# output file
# new entries will be appended, if the file already exists
filename = "properwatcher.csv"
# especially useful if used as aws lambda function
# simply permit access to dynamodb for the lambda function's role
# if not run as lambda, you need to set environment variables:
# - AWS_ACCESS_KEY_ID
# - AWS_SECRET_ACCESS_KEY
[dynamodb]
# name of the table in your dynamodb
table_name = "properties"
# the region your dynamodb is located in
region = "eu-central-1"
# found properties can be filtered for standard attributes
# only properties matching the given criteria will be processed
[criteria]
price_min = 100
price_max = 500
squaremeters_min = 10
squaremeters_max = 60
rooms_min = 1
rooms_max = 2