-
Notifications
You must be signed in to change notification settings - Fork 2
/
extract_special_predicates.py
70 lines (63 loc) · 2.85 KB
/
extract_special_predicates.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
"""
Extract identifier predicates and geo predicates.
Such predicates (and corresponding facts) are removed from the dump.
"""
import pickle
import sys
#####################################################
# CONSTANTS #
#####################################################
PATH_TO_EXT_ID_PREDICATES = "dicts/identifier_predicates.pickle"
PATH_TO_GEO_PREDICATES = "dicts/geo_predicates.pickle"
#####################################################
# FUNCTIONS #
#####################################################
def extract_special_predicates(file, ext_id_predicates, geo_predicates):
"""
Extract special (unnecessary) predicates from the wikidata dump.
"""
with open(file, "r") as fp:
line = fp.readline()
while line:
currentLine = line
s, p, o = currentLine.split(" ", 2)
line = fp.readline()
# Remove " .\n", the line ending from o
if len(o) < 35:
continue
o = o[:-3]
# Extract external identifier predicates
if p == "<http://wikiba.se/ontology#propertyType>" and o == "<http://wikiba.se/ontology#ExternalId>":
ext_id_pred = s.rsplit("/", 1)[1].replace(">", "")
if not ext_id_pred in ext_id_predicates:
ext_id_predicates.append(ext_id_pred)
# Extract geo predicates (for geo shapes)
if p == "<http://wikiba.se/ontology#propertyType>" and o == "<http://wikiba.se/ontology#GeoShape>":
geo_pred = s.rsplit("/", 1)[1].replace(">", "")
if not geo_pred in geo_predicates:
geo_predicates.append(geo_pred)
# Extract geo predicates (for geo coordinates)
if p == "<http://wikiba.se/ontology#propertyType>" and o == "<http://wikiba.se/ontology#GlobeCoordinate>":
geo_pred = s.rsplit("/", 1)[1].replace(">", "")
if not geo_pred in geo_predicates:
geo_predicates.append(geo_pred)
def store_special_predicates(special_predicates, path):
"""
Store the special predicates on the given path.
Return: None
"""
with open(path, "wb") as output:
pickle.dump(special_predicates, output, protocol=pickle.HIGHEST_PROTOCOL)
#####################################################
# MAIN #
#####################################################
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: extract_identifier_predicates.py <wikidata_dump_path>")
sys.exit()
WIKIDATA_DUMP_PATH = sys.argv[1]
ext_id_predicates = list()
geo_predicates = list()
extract_special_predicates(WIKIDATA_DUMP_PATH, ext_id_predicates, geo_predicates)
store_special_predicates(ext_id_predicates, PATH_TO_EXT_ID_PREDICATES)
store_special_predicates(geo_predicates, PATH_TO_GEO_PREDICATES)