Skip to content

Commit

Permalink
V58
Browse files Browse the repository at this point in the history 
- Nevenadressen are now parsed. They are stored in a nevenadressen table.
- Adressen table has a new hoofd_nummer_id field. If filled, points to the hoofdadres.
- Parser now handles multiple field values. For example, a verblijfsobject can contain multiple nevenadressen.
- Updated progress bar data
- Renamed "openbareruimte_id" field in nummers table to "openbare_ruimte_id"
- More logging when creating adressen table.
  • Loading branch information
@digitaldutch
digitaldutch committed Jul 4, 2023
1 parent dffa731 commit a7ecfd0
Show file tree
Hide file tree
Showing 5 changed files with 155 additions and 76 deletions.
43 changes: 24 additions & 19 deletions bag/bag_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def parse(self, tag_name):

self.object_tag_name = ns_objecten + tag_name
self.file_bag_code = "9999WPL"
self.total_xml = 3889 # required for progress indicator. Actual numbers can be found in the console or log.
self.total_xml = 4049 # required for progress indicator. Actual numbers can be found in the console or log.
self.data_init = {'geometry': ''}
self.save_to_database = self.__save_woonplaats
self.db_fields = {
Expand All @@ -65,7 +65,7 @@ def parse(self, tag_name):

self.object_tag_name = ns_gwr_product + tag_name
self.file_bag_code = "GEM-WPL-RELATIE"
self.total_xml = 5765 # required for progress indicator
self.total_xml = 5773 # required for progress indicator
self.save_to_database = self.__save_gemeente_woonplaats

self.db_fields = {
Expand All @@ -86,9 +86,9 @@ def parse(self, tag_name):

self.object_tag_name = ns_objecten + tag_name
self.file_bag_code = "9999OPR"
self.total_xml = 343448 # required for progress indicator
self.total_xml = 346970 # required for progress indicator
self.data_init = {'verkorte_naam': ''}
self.save_to_database = self.__save_openbareruimte
self.save_to_database = self.__save_openbare_ruimte

self.db_fields = {
ns_objecten + 'identificatie': 'id',
Expand All @@ -109,7 +109,7 @@ def parse(self, tag_name):

self.object_tag_name = ns_objecten + tag_name
self.file_bag_code = "9999NUM"
self.total_xml = 12054045 # required for progress indicator
self.total_xml = 12287165 # required for progress indicator
# Initialization required as BAG leaves fields out of the data if it is empty
self.data_init = {'huisletter': '', 'toevoeging': '', 'postcode': '', 'woonplaats_id': ''}
self.save_to_database = self.__save_nummer
Expand All @@ -123,7 +123,7 @@ def parse(self, tag_name):
ns_historie + 'beginGeldigheid': 'begindatum_geldigheid',
ns_historie + 'eindGeldigheid': 'einddatum_geldigheid',
ns_objecten + 'status': 'status',
ns_objecten_ref + 'OpenbareRuimteRef': 'openbareruimte_id',
ns_objecten_ref + 'OpenbareRuimteRef': 'openbare_ruimte_id',
ns_objecten_ref + 'WoonplaatsRef': 'woonplaats_id',
}
self.db_tag_parent_fields = {}
Expand All @@ -134,7 +134,7 @@ def parse(self, tag_name):

self.object_tag_name = ns_objecten + tag_name
self.file_bag_code = "9999PND"
self.total_xml = 20352252 # required for progress indicator
self.total_xml = 21286109 # required for progress indicator
self.data_init = {'geometry': ''}
self.save_to_database = self.__save_pand

Expand All @@ -157,8 +157,8 @@ def parse(self, tag_name):

self.object_tag_name = ns_objecten + tag_name
self.file_bag_code = "9999VBO"
self.total_xml = 21148447 # required for progress indicator
self.data_init = {'pos': '', 'rd_x': '', 'rd_y': '', 'latitude': '', 'longitude': ''}
self.total_xml = 22552963 # required for progress indicator
self.data_init = {'pos': '', 'rd_x': '', 'rd_y': '', 'latitude': '', 'longitude': '', 'nevenadressen': ''}
self.save_to_database = self.__save_verblijfsobject

self.db_fields = {
Expand All @@ -175,6 +175,7 @@ def parse(self, tag_name):
# Therefore, identification is done by combining the tag with the parent tag
self.db_tag_parent_fields = {
ns_objecten + 'heeftAlsHoofdadres' + ns_objecten_ref + 'NummeraanduidingRef': 'nummer_id',
ns_objecten + 'heeftAlsNevenadres' + ns_objecten_ref + 'NummeraanduidingRef': 'nevenadressen',
}
elif self.tag_name == 'Ligplaats':
ns_objecten = "{www.kadaster.nl/schemas/lvbag/imbag/objecten/v20200601}"
Expand All @@ -184,7 +185,7 @@ def parse(self, tag_name):

self.object_tag_name = ns_objecten + tag_name
self.file_bag_code = "9999LIG"
self.total_xml = 17653 # required for progress indicator
self.total_xml = 18131 # required for progress indicator
self.data_init = {'pos': '', 'rd_x': '', 'rd_y': '', 'latitude': '', 'longitude': '', 'geometry': ''}
self.save_to_database = self.__save_ligplaats

Expand All @@ -210,7 +211,7 @@ def parse(self, tag_name):

self.object_tag_name = ns_objecten + tag_name
self.file_bag_code = "9999STA"
self.total_xml = 49543 # required for progress indicator
self.total_xml = 56684 # required for progress indicator
self.data_init = {'pos': '', 'rd_x': '', 'rd_y': '', 'latitude': '', 'longitude': '', 'geometry': ''}
self.save_to_database = self.__save_standplaats

Expand Down Expand Up @@ -279,13 +280,14 @@ def __update_status(self, final=False):

def __parse_file(self, file_xml):
data = self.data_init.copy()
tag_previous = None
tag_now = None
parent_tags = []

for event, elem in ElementTree.iterparse(file_xml, events=("start", "end")):
if event == 'start':
tag_previous = tag_now
tag_now = elem.tag
parent_tags.append(elem.tag)
elif event == 'end':
parent_tags.pop()

# Note: elem.text is only guaranteed in 'end' event
if elem.tag == self.object_tag_name:
self.count_xml += 1
Expand All @@ -295,11 +297,14 @@ def __parse_file(self, file_xml):
else:
field_found = False

if self.db_tag_parent_fields and tag_previous:
parent_elem_tag = tag_previous + elem.tag
if self.db_tag_parent_fields and parent_tags:
parent_elem_tag = parent_tags[-1] + elem.tag
field_parent_elem = self.db_tag_parent_fields.get(parent_elem_tag)
if field_parent_elem:
data[field_parent_elem] = elem.text
if field_parent_elem in data and data[field_parent_elem]:
data[field_parent_elem] += "," + elem.text
else:
data[field_parent_elem] = elem.text
field_found = True

if not field_found:
Expand All @@ -322,7 +327,7 @@ def __save_gemeente_woonplaats(self, data):
self.__update_status()
self.database.save_gemeente_woonplaats(data)

def __save_openbareruimte(self, data):
def __save_openbare_ruimte(self, data):
if (self.__bag_einddatum_valid(data) and
self.__bag_begindatum_valid(data) and
data['status'] == "Naamgeving uitgegeven"):
Expand Down
6 changes: 3 additions & 3 deletions config.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import locale

version = 56
version_date = '20 June 2023'
version = 58
version_date = '4 July 2023'

locale.setlocale(locale.LC_ALL, 'nl_NL')

Expand All @@ -23,7 +23,7 @@

# If an adressen table is created some BAG tables are no longer needed and can be deleted:
# nummers, panden, verblijfsobjecten, ligplaatsen and standplaatsen. Set to False if you want to keep these tables.
# You can also delete these tables afterwards using the utils_sqlite_shrink.py script.
# You can also delete these tables afterward using the utils_sqlite_shrink.py script.
delete_no_longer_needed_bag_tables = True

# Public spaces with names longer than 24 characters also have a shortened name. Set to true to make short names the
Expand Down
133 changes: 98 additions & 35 deletions database_sqlite/database_sqlite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,32 +71,40 @@ def save_openbare_ruimte(self, data):
def save_nummer(self, data):
# Note: Use replace, because BAG does not always contain unique id's
self.connection.execute(
"""REPLACE INTO nummers (id, postcode, huisnummer, huisletter, toevoeging, woonplaats_id, openbareruimte_id,
status) VALUES(?, ?, ?, ?, ?, ?, ?, ?);
"""REPLACE INTO nummers (id, postcode, huisnummer, huisletter, toevoeging, woonplaats_id,
openbare_ruimte_id, status) VALUES(?, ?, ?, ?, ?, ?, ?, ?);
""",
(data["id"], data["postcode"], data["huisnummer"], data["huisletter"], data["toevoeging"],
data["woonplaats_id"], data["openbareruimte_id"], data["status"])
data["woonplaats_id"], data["openbare_ruimte_id"], data["status"])
)


def save_pand(self, data):
# Note: Use replace, because BAG does not always contain unique id's
self.connection.execute(
"""REPLACE INTO panden (id, bouwjaar, status, geometry)
VALUES(?, ?, ?, ?)
""",
(data["id"], data["bouwjaar"], data["status"], data["geometry"]))
(data["id"], data["bouwjaar"], data["status"], data["geometry"])
)

def save_verblijfsobject(self, data):
# Note: Use replace, because BAG does not always contain unique id's
self.connection.execute(
"""REPLACE INTO verblijfsobjecten (id, nummer_id, pand_id, oppervlakte, rd_x, rd_y, latitude, longitude,
gebruiksdoel, status) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
gebruiksdoel, status) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
""",
(data["id"], data["nummer_id"], data["pand_id"], data["oppervlakte"], data["rd_x"], data["rd_y"],
data["latitude"], data["longitude"], data["gebruiksdoel"], data["status"])
)

if data["nevenadressen"]:
nevenadressen = data["nevenadressen"].split(",")
for neven_nummer_id in nevenadressen:
self.connection.execute("""
INSERT INTO nevenadressen (verblijfsobject_id, neven_nummer_id, hoofd_nummer_id) VALUES (?, ?, ?);
""", (data["id"], neven_nummer_id, data["nummer_id"])
)

def save_ligplaats(self, data):
# Note: Use replace, because BAG does not always contain unique id's
self.connection.execute(
Expand Down Expand Up @@ -130,27 +138,35 @@ def create_bag_tables(self):
DROP TABLE IF EXISTS openbare_ruimten;
CREATE TABLE openbare_ruimten (id INTEGER PRIMARY KEY, naam TEXT, lange_naam TEXT, verkorte_naam TEXT,
type TEXT, woonplaats_id INTEGER);
type TEXT, woonplaats_id INTEGER);
DROP TABLE IF EXISTS nummers;
CREATE TABLE nummers (id TEXT PRIMARY KEY, postcode TEXT, huisnummer INTEGER, huisletter TEXT,
toevoeging TEXT, woonplaats_id TEXT, openbareruimte_id TEXT, status TEXT);
toevoeging TEXT, woonplaats_id TEXT, openbare_ruimte_id TEXT, status TEXT);
DROP TABLE IF EXISTS panden;
CREATE TABLE panden (id TEXT PRIMARY KEY, bouwjaar INTEGER, status TEXT, geometry TEXT);
DROP TABLE IF EXISTS verblijfsobjecten;
CREATE TABLE verblijfsobjecten (id TEXT PRIMARY KEY, nummer_id TEXT, pand_id TEXT,
oppervlakte FLOAT, rd_x FLOAT, rd_y FLOAT, latitude FLOAT, longitude FLOAT, gebruiksdoel TEXT,
status TEXT);
oppervlakte FLOAT, rd_x FLOAT, rd_y FLOAT, latitude FLOAT, longitude FLOAT, gebruiksdoel TEXT,
status TEXT);
DROP TABLE IF EXISTS nevenadressen;
CREATE TABLE nevenadressen (
verblijfsobject_id TEXT,
neven_nummer_id TEXT,
hoofd_nummer_id TEXT,
PRIMARY KEY (verblijfsobject_id, neven_nummer_id)
);
DROP TABLE IF EXISTS ligplaatsen;
CREATE TABLE ligplaatsen (id TEXT PRIMARY KEY, nummer_id TEXT, rd_x FLOAT, rd_y FLOAT, latitude FLOAT,
longitude FLOAT, status TEXT, geometry TEXT);
longitude FLOAT, status TEXT, geometry TEXT);
DROP TABLE IF EXISTS standplaatsen;
CREATE TABLE standplaatsen (id TEXT PRIMARY KEY, nummer_id TEXT, rd_x FLOAT, rd_y FLOAT, latitude FLOAT,
longitude FLOAT, status TEXT, geometry TEXT);
longitude FLOAT, status TEXT, geometry TEXT);
""")
self.connection.commit()

Expand All @@ -173,48 +189,60 @@ def create_indices_adressen(self):

def create_adressen_from_bag(self):

utils.print_log('create adressen tabel: import adressen')
self.connection.executescript(f"""
DROP TABLE IF EXISTS adressen;
CREATE TABLE adressen (nummer_id TEXT PRIMARY KEY, pand_id TEXT, verblijfsobject_id TEXT,
gemeente_id INTEGER, woonplaats_id INTEGER, openbare_ruimte_id INTEGER, object_type TEXT,
gebruiksdoel TEXT, postcode TEXT, huisnummer INTEGER, huisletter TEXT, toevoeging TEXT,
oppervlakte FLOAT, rd_x FLOAT, rd_y FLOAT, latitude FLOAT, longitude FLOAT, bouwjaar INTEGER,
oppervlakte FLOAT, rd_x FLOAT, rd_y FLOAT, latitude FLOAT, longitude FLOAT, bouwjaar INTEGER,
hoofd_nummer_id TEXT,
geometry TEXT);
INSERT INTO adressen (nummer_id, pand_id, verblijfsobject_id, gemeente_id, woonplaats_id,
openbare_ruimte_id, object_type, gebruiksdoel, postcode, huisnummer, huisletter, toevoeging,
oppervlakte, rd_x, rd_y, longitude, latitude, bouwjaar, geometry)
SELECT
n.id AS nummer_id,
p.id AS pand_id,
v.id AS verblijfsobject_id,
w.gemeente_id,
o.woonplaats_id,
o.id,
'verblijfsobject',
v.gebruiksdoel,
n.postcode,
n.huisnummer,
n.huisletter,
n.toevoeging,
v.oppervlakte,
v.rd_x,
v.rd_y,
v.longitude,
v.latitude,
p.bouwjaar,
p.geometry
n.id AS nummer_id,
p.id AS pand_id,
v.id AS verblijfsobject_id,
w.gemeente_id,
o.woonplaats_id,
o.id,
'verblijfsobject',
v.gebruiksdoel,
n.postcode,
n.huisnummer,
n.huisletter,
n.toevoeging,
v.oppervlakte,
v.rd_x,
v.rd_y,
v.longitude,
v.latitude,
p.bouwjaar,
p.geometry
FROM nummers n
LEFT JOIN openbare_ruimten o ON o.id = n.openbareruimte_id
LEFT JOIN openbare_ruimten o ON o.id = n.openbare_ruimte_id
LEFT JOIN woonplaatsen w ON w.id = o.woonplaats_id
LEFT JOIN verblijfsobjecten v ON v.nummer_id = n.id
LEFT JOIN panden p ON v.pand_id = p.id;
""")

utils.print_log('create adressen tabel: import ligplaatsen data')
self.adressen_import_ligplaatsen()

utils.print_log('create adressen tabel: import standplaatsen data')
self.adressen_import_standplaatsen()

utils.print_log('create adressen tabel: import woonplaatsen from nummers')
self.adressen_update_woonplaatsen_from_nummers()

utils.print_log('create adressen tabel: update nevenadressen data')
self.adressen_update_nevenadressen()

utils.print_log('create adressen tabel: create indices')
self.create_indices_adressen()

self.connection.commit()
Expand Down Expand Up @@ -245,7 +273,41 @@ def adressen_import_standplaatsen(self):
WHERE s.nummer_id = adressen.nummer_id;
""")

# woonplaats_id in nummers overruled woonplaats_id van de openbare ruimte.
def adressen_update_nevenadressen(self):
self.connection.executescript("""
UPDATE adressen SET
hoofd_nummer_id = n.hoofd_nummer_id,
pand_id = n.pand_id,
verblijfsobject_id = n.verblijfsobject_id,
gebruiksdoel = n.gebruiksdoel,
oppervlakte = n.oppervlakte,
rd_x = n.rd_x,
rd_y = n.rd_y,
latitude = n.latitude,
longitude = n.longitude,
bouwjaar = n.bouwjaar,
geometry = n.geometry
FROM (
SELECT
nevenadressen.hoofd_nummer_id,
nevenadressen.neven_nummer_id,
adressen.pand_id,
adressen.verblijfsobject_id,
adressen.gebruiksdoel,
adressen.oppervlakte,
adressen.rd_x,
adressen.rd_y,
adressen.latitude,
adressen.longitude,
adressen.bouwjaar,
adressen.geometry
FROM nevenadressen
LEFT JOIN adressen ON nevenadressen.hoofd_nummer_id = adressen.nummer_id
) AS n
WHERE n.neven_nummer_id = adressen.nummer_id;
""")

# woonplaats_id in nummers overrule woonplaats_id van de openbare ruimte.
def adressen_update_woonplaatsen_from_nummers(self):
self.connection.executescript("""
UPDATE adressen SET
Expand All @@ -258,6 +320,7 @@ def delete_no_longer_needed_bag_tables(self):
self.connection.executescript("""
DROP TABLE IF EXISTS nummers;
DROP TABLE IF EXISTS panden;
DROP TABLE IF EXISTS nevenadressen;
DROP TABLE IF EXISTS verblijfsobjecten;
DROP TABLE IF EXISTS ligplaatsen;
DROP TABLE IF EXISTS standplaatsen;
Expand Down Expand Up @@ -344,7 +407,7 @@ def test_bag_adressen(self):

# Het is makkelijk om per ongeluk een gemeenten.csv te genereren die niet in UTF-8 is. Testen dus.
naam = self.fetchone("SELECT naam FROM gemeenten WHERE id=1900")
utils.print_log("test: Gemeentenamen moeten in UTF-8 zijn: " + naam, naam != 'Súdwest-Fryslân')
utils.print_log("test: gemeentenamen moeten in UTF-8 zijn: " + naam, naam != 'Súdwest-Fryslân')

aantal = self.fetchone("SELECT COUNT(*) FROM adressen WHERE adressen.latitude IS NULL AND pand_id IS NOT NULL;")
utils.print_log("test: panden zonder locatie: " + str(aantal), aantal > 0)
Expand Down
1 change: 0 additions & 1 deletion import_bag.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
db_sqlite.create_indices_bag()

if config. create_adressen_table:
utils.print_log('create adressen tabel')
db_sqlite.create_adressen_from_bag()
db_sqlite.adressen_fix_bag_errors()
db_sqlite.test_bag_adressen()
Expand Down
Loading

0 comments on commit a7ecfd0

Please sign in to comment.