From 7ea3f423758bb14a27e9071cab8a3231bfcd1a17 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sun, 13 Nov 2022 22:17:31 +0100 Subject: [PATCH] [FEAT] save timestamp of last-changed of raw map files being processed doing #162 with the timestamp of downloaded maps --- wahoomc/osm_maps_functions.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index b5f161d0..7b36ad2c 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -4,6 +4,7 @@ #!/usr/bin/python # import official python packages +from datetime import datetime import glob import multiprocessing import os @@ -103,6 +104,15 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""): sys.exit() +def get_timestamp_last_changed(file_path): + """ + returns the timestamp of the last-changed datetime of the given file + """ + chg_time = os.path.getmtime(file_path) + + return datetime.fromtimestamp(chg_time).isoformat() + + class OsmData(): # pylint: disable=too-few-public-methods """ object with all internal parameters to process maps @@ -351,7 +361,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st # - force processing is set (this is also when new map files were dowwnloaded) # - the defined TAGS_TO_KEEP_UNIVERSAL constants have changed are changed (user input or new release) if not os.path.isfile(out_file_o5m_filtered_win) or not os.path.isfile(out_file_o5m_filtered_names_win) \ - or self.o_osm_data.force_processing is True or self.tags_are_identical_to_last_run(key) is False: + or self.o_osm_data.force_processing is True or self.tags_are_identical_to_last_run(key) is False \ + or self.last_changed_is_identical_to_last_run(key) is False: log.info( '+ Filtering unwanted map objects out of map of %s', key) cmd = [get_tooling_win_path(['osmfilter'])] @@ -391,7 +402,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st # - force processing is set (this is also when new map files were dowwnloaded) # - the defined TAGS_TO_KEEP_UNIVERSAL constants have changed are changed (user input or new release) if not os.path.isfile(out_file_pbf_filtered_mac) or not os.path.isfile(out_file_pbf_filtered_names_mac) \ - or self.o_osm_data.force_processing is True or self.tags_are_identical_to_last_run(key) is False: + or self.o_osm_data.force_processing is True or self.tags_are_identical_to_last_run(key) is False \ + or self.last_changed_is_identical_to_last_run(key) is False: log.info( '+ Filtering unwanted map objects out of map of %s', key) @@ -858,6 +870,7 @@ def write_country_config_file(self, country): # Data to be written configuration = { "version_last_run": VERSION, + "changed_ts_map_last_run": get_timestamp_last_changed(self.o_osm_data.border_countries[country]['map_file']), "tags_last_run": translate_tags_to_keep(sys_platform=platform.system()), "name_tags_last_run": translate_tags_to_keep(name_tags=True, sys_platform=platform.system()) } @@ -881,3 +894,19 @@ def tags_are_identical_to_last_run(self, country): tags_are_identical = False return tags_are_identical + + def last_changed_is_identical_to_last_run(self, country): + """ + compare tags of this run with used tags from last run stored in _tiles/{country} directory + """ + last_changed_is_identical = True + + try: + country_config = read_json_file(os.path.join( + USER_OUTPUT_DIR, country, ".config.json")) + if not country_config["changed_ts_map_last_run"] == get_timestamp_last_changed(self.o_osm_data.border_countries[country]['map_file']): + last_changed_is_identical = False + except (FileNotFoundError, KeyError): + last_changed_is_identical = False + + return last_changed_is_identical