-
Notifications
You must be signed in to change notification settings - Fork 0
/
etl-bmkg.py
86 lines (76 loc) · 2.24 KB
/
etl-bmkg.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
from kamus import kamus_cuaca
import bonobo
import requests as req
import xmltodict
import json
def input_data():
_input = input("Masukan Nama Wilayah : ")
return _input
def scrape_bmkg(wilayah):
try:
print("scrape ",wilayah)
#wilayah = 'DIYogyakarta'
url = 'http://data.bmkg.go.id/datamkg/MEWS/DigitalForecast/DigitalForecast-'+wilayah+'.xml'
body = req.get(url)
parse_data = xmltodict.parse(body.text,attr_prefix='')
print(parse_data)
hasil = json.dumps(parse_data)
final = json.loads(hasil)
dt = final['data']['forecast']
return dt, wilayah
except :
return None
def extract(x):
try:
print("extract ",x)
yield scrape_bmkg(x)[0]
except :
print("Data Tidak Ditemukan")
def transform(dt):
try:
dCuaca = kamus_cuaca()
_data = []
for i in dt['area']:
x = i['domain']
weather = []
area = {
'source': 'BMKG (Badan Meteorologi, Klimatologi, dan Geofisika)',
'id': i['id'],
'latitude': i['latitude'],
'longitude': i['longitude'],
'coordinate': i['coordinate'],
'kabupaten': i['description'],
'propinsi': i['domain'],
'waktu': dt['issue']
}
for x in i['parameter']:
if x['id'] == 'weather':
for y in x['timerange']:
kd = y['value']['#text']
ketCuaca = dCuaca[int(kd)]
y['value']['#text'] = ketCuaca
weather.append(y)
temp = {
'weather':weather
}
area.update(temp)
_data.append(area)
yield _data
except :
None
def load(xyz):
try:
nama = xyz[0]['propinsi']
nama = nama.replace(' ','')
with open(nama+'.json', 'w') as fp:
json.dump(xyz, fp)
except :
print("Tidak Dapat Menampilkan Hasil")
if __name__ == '__main__':
x = input_data()
graph = bonobo.Graph(
extract(x),
transform,
load
)
bonobo.run(graph)