From 3026e8b4dffc433654f042a0d3680d4622aabedb Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 19:14:58 +0200 Subject: [PATCH 01/17] update pylint to v3.3.* Remove the now redundant controls, found with: pylint --enable=useless-suppression --- .github/workflows/tests.yml | 2 +- .pylintrc | 3 --- conda_env/gdal-dev.yml | 2 +- wahoomc/osm_maps_functions.py | 8 +++----- wahoomc/setup_functions.py | 2 +- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 70e9ffe4..cea4432c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pylint==2.15.* + pip install pylint==3.3.* pip install mock pip install requests==2.28.* - name: Analysing the code with pylint diff --git a/.pylintrc b/.pylintrc index a95f775e..22dd6143 100644 --- a/.pylintrc +++ b/.pylintrc @@ -17,6 +17,3 @@ disable=line-too-long, duplicate-code ; [MASTER] ; init-hook='import sys; sys.path.append("/path/to/root")' ; init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))" - -[MASTER] -init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))" diff --git a/conda_env/gdal-dev.yml b/conda_env/gdal-dev.yml index c20086aa..08333e8b 100644 --- a/conda_env/gdal-dev.yml +++ b/conda_env/gdal-dev.yml @@ -5,7 +5,7 @@ dependencies: - python=3.10 - gdal=3.6.* - requests=2.28.* - - pylint=2.15.* + - pylint=3.3.* - geojson=2.5.* - shapely=1.8.* - osmium-tool=1.16.* diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 94128805..e6fc9ffc 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -43,9 +43,7 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""): cmd, capture_output=True, text=True, encoding="utf-8", check=False) else: - process = subprocess.run( # pylint: disable=consider-using-with - cmd, capture_output=True, cwd=cwd, text=True, encoding="utf-8", check=False) - + process = subprocess.run(cmd, capture_output=True, cwd=cwd, text=True, encoding="utf-8", check=False) if error_message and process.returncode != 0: # 0 means success log.error('subprocess error output:') @@ -450,7 +448,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto log.info('# Merge splitted tiles with land, elevation, and sea') timings = Timings() tile_count = 1 - for tile in self.o_osm_data.tiles: # pylint: disable=too-many-nested-blocks + for tile in self.o_osm_data.tiles: self.log_tile_info(tile["x"], tile["y"], tile_count) timings_tile = Timings() @@ -785,7 +783,7 @@ def log_tile_debug(self, tile_x, tile_y, tile_count, additional_info=''): """ self.log_tile(tile_x, tile_y, tile_count, True, additional_info) - def log_tile(self, tile_x, tile_y, tile_count, log_level_debug, additional_info=''): # pylint: disable=too-many-arguments + def log_tile(self, tile_x, tile_y, tile_count, log_level_debug, additional_info=''): # pylint: disable=too-many-arguments,too-many-positional-arguments """ unified status logging for this class """ diff --git a/wahoomc/setup_functions.py b/wahoomc/setup_functions.py index ad735ed7..7d38e66b 100644 --- a/wahoomc/setup_functions.py +++ b/wahoomc/setup_functions.py @@ -45,7 +45,7 @@ def adjustments_due_to_breaking_changes(): """ handle breaking changes """ - version_last_run = read_version_last_run() # pylint: disable=unused-variable + version_last_run = read_version_last_run() # Osmosis in v.0.49.2 seams not to be working on WINDOWS since the upgrade to v0.49.2 # - due to the path into it was downloaded, 'tooling_win/Osmosis/osmosis-0.49.2' From 270668506389818a5f6b0d6a3d1f03ac2cdf6ab1 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 19:15:48 +0200 Subject: [PATCH 02/17] fix pylint possibly-used-before-assignment warning is_required_input_given_or_exit() already checks that either country or x/y is set, make pylint happy with a simple "else". Fixes: wahoomc/main.py:76:8: E0606: Possibly using variable 'o_osm_data' before assignment (possibly-used-before-assignment) --- tests/test_osm_maps.py | 3 ++- wahoomc/main.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index 9eb8db04..73008f23 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -148,7 +148,8 @@ def process_and_check_border_countries(self, inp_val, calc_border_c, exp_result, if inp_mode == 'country': o_input_data.country = inp_val o_osm_data = CountryOsmData(o_input_data) - elif inp_mode == 'xy_coordinate': + else: + self.assertEqual(inp_mode, 'xy_coordinate') o_input_data.xy_coordinates = inp_val o_osm_data = XYOsmData(o_input_data) diff --git a/wahoomc/main.py b/wahoomc/main.py index dfa17b11..79ce9a37 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -61,7 +61,7 @@ def run(run_level): if o_input_data.country: o_osm_data = CountryOsmData(o_input_data) - elif o_input_data.xy_coordinates: + else: o_osm_data = XYOsmData(o_input_data) timings = Timings() From 0408165c33ece7d985009661122d8df056bb2838 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Mon, 19 May 2025 10:49:49 +0200 Subject: [PATCH 03/17] use the current python interpreter for additional processes This makes it work everywhere, no matter if it's `python` or `python3`. --- tests/test_cli.py | 7 ++++--- tests/test_generated_files.py | 5 +++-- wahoomc/osm_maps_functions.py | 12 ++---------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index c2723198..aa595524 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,7 @@ tests for the python file """ import os +import sys import unittest # import custom python packages @@ -17,7 +18,7 @@ def test_top_parser_help(self): tests, if help of top parser can be called """ - result = os.system("python -m wahoomc -h") + result = os.system(sys.executable + " -m wahoomc -h") self.assertEqual(result, 0) @@ -26,7 +27,7 @@ def test_cli_help(self): tests, if CLI help can be called """ - result = os.system("python -m wahoomc cli -h") + result = os.system(sys.executable + " -m wahoomc cli -h") self.assertEqual(result, 0) @@ -35,7 +36,7 @@ def test_gui_help(self): tests, if GUI help can be called """ - result = os.system("python -m wahoomc gui -h") + result = os.system(sys.executable + " -m wahoomc gui -h") self.assertEqual(result, 0) diff --git a/tests/test_generated_files.py b/tests/test_generated_files.py index 92c8b9e4..08f563eb 100644 --- a/tests/test_generated_files.py +++ b/tests/test_generated_files.py @@ -7,6 +7,7 @@ from os import walk import platform import shutil +import sys import unittest import subprocess @@ -214,11 +215,11 @@ def run_wahoomapscreator_cli(self, country, hdd_mode=False): if not hdd_mode: # run processing of input-country via CLI in standard mode result = os.system( - f'python -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc') + f'{sys.executable} -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc') else: # run processing of input-country via CLI in mapwriter hdd mode result = os.system( - f'python -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc -hd') + f'{sys.executable} -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc -hd') # check if run was successful self.assertEqual(result, 0) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index e6fc9ffc..c866d8a6 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -239,16 +239,8 @@ def generate_land(self): # create land1.osm if not os.path.isfile(out_file_land1+'1.osm') or self.o_osm_data.force_processing is True: - # Windows - if platform.system() == "Windows": - cmd = ['python', os.path.join(RESOURCES_DIR, - 'shape2osm.py'), '-l', out_file_land1, land_file] - - # Non-Windows - else: - cmd = ['python', os.path.join(RESOURCES_DIR, - 'shape2osm.py'), '-l', out_file_land1, land_file] - + cmd = [sys.executable, os.path.join(RESOURCES_DIR, + 'shape2osm.py'), '-l', out_file_land1, land_file] run_subprocess_and_log_output( cmd, f'! Error creating land.osm for tile: {tile["x"]},{tile["y"]}') self.log_tile_debug(tile["x"], tile["y"], tile_count, timings_tile.stop_and_return()) From bf0cfe7b349c86debf8e76c2328377819afb3298 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Sun, 25 May 2025 08:29:24 +0200 Subject: [PATCH 04/17] catch JSONDecodeError in get_latest_pypi_version() This actually happened, in which case everything falls apart. --- wahoomc/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index bed2d948..338f4fc4 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -180,7 +180,7 @@ def get_latest_pypi_version(): response = requests.get( 'https://pypi.org/pypi/wahoomc/json', timeout=1) return response.json()['info']['version'] - except (requests.ConnectionError, requests.Timeout): + except (requests.ConnectionError, requests.Timeout, requests.exceptions.JSONDecodeError): return None From dc5057524d66511b7018dd0dbb0ac71c6477c377 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 20:27:31 +0200 Subject: [PATCH 05/17] fix "-hdd" typo in test_generated_files.py --- tests/test_generated_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_generated_files.py b/tests/test_generated_files.py index 08f563eb..aac7dffb 100644 --- a/tests/test_generated_files.py +++ b/tests/test_generated_files.py @@ -219,7 +219,7 @@ def run_wahoomapscreator_cli(self, country, hdd_mode=False): else: # run processing of input-country via CLI in mapwriter hdd mode result = os.system( - f'{sys.executable} -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc -hd') + f'{sys.executable} -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc -hdd') # check if run was successful self.assertEqual(result, 0) From 99798c55419138334edaaaa090a4fe3b0ec6e6f1 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Mon, 2 Jun 2025 18:58:13 +0200 Subject: [PATCH 06/17] log full subprocess command lines Visible when using --verbose. --- wahoomc/osm_maps_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index c866d8a6..2dd0a886 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -38,6 +38,7 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""): """ run given cmd-subprocess and issue error message if wished """ + log.debug('running subprocess: %s', str(cmd)) if not cwd: process = subprocess.run( cmd, capture_output=True, text=True, encoding="utf-8", check=False) From 51c15827bdc5a39c500755a902f761f64463dffb Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Mon, 2 Jun 2025 18:57:24 +0200 Subject: [PATCH 07/17] make translate_tags_to_keep() platform independent Pass on which tool to use, osmfilter or osmium. This makes it easier to switch to the other tool, as all are available on all platforms. --- tests/test_constants.py | 10 ++++------ wahoomc/constants_functions.py | 20 ++++++-------------- wahoomc/osm_maps_functions.py | 28 ++++++++++------------------ 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/tests/test_constants.py b/tests/test_constants.py index 8c6c5e82..1c9b74d7 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -51,7 +51,7 @@ def test_translate_tags_to_keep_simple_win(self, mock_open, mock_json_load): # tags_win = 'access= area=yes' mock_json_load.return_value = tags_universal_simple - transl_tags = translate_tags_to_keep(sys_platform='Windows') + transl_tags = translate_tags_to_keep(osmium=False) self.assertEqual(tags_win, transl_tags) @ mock.patch("wahoomc.file_directory_functions.json.load") @@ -76,7 +76,7 @@ def test_translate_tags_to_keep_adv_win(self, mock_open, mock_json_load): # pyl tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated' mock_json_load.return_value = tags_universal_adv - transl_tags = translate_tags_to_keep(sys_platform='Windows') + transl_tags = translate_tags_to_keep(osmium=False) self.assertEqual(tags_win, transl_tags) def test_translate_tags_to_keep_full_macos(self): @@ -100,8 +100,7 @@ def test_translate_tags_to_keep_full_win(self): """ tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated amenity=fuel =cafe =drinking_water =shelter shop=bakery =bicycle highway=abandoned =bus_guideway =disused =bridleway =byway =construction =cycleway =footway =living_street =motorway =motorway_link =path =pedestrian =primary =primary_link =residential =road =secondary =secondary_link =service =steps =tertiary =tertiary_link =track =trunk =trunk_link =unclassified natural=coastline =nosea =sea =beach =land =scrub =water =wetland =wood landuse=forest =commercial =industrial =residential =retail leisure=park =nature_reserve railway=rail =tram =station =stop surface= tracktype= tunnel= waterway=canal =drain =river =riverbank =stream wood=deciduous tourism=alpine_hut' - transl_tags = translate_tags_to_keep( - sys_platform='Windows', use_repo=True) + transl_tags = translate_tags_to_keep(osmium=False, use_repo=True) self.assertEqual(tags_win, transl_tags) def test_translate_name_tags_to_keep_full_macos(self): @@ -121,8 +120,7 @@ def test_translate_name_tags_to_keep_full_win(self): names_tags_win = 'admin_level=2 area=yes mountain_pass= natural= place=city =hamlet =island =isolated_dwelling =islet =locality =suburb =town =village =country' - transl_tags = translate_tags_to_keep( - name_tags=True, sys_platform='Windows', use_repo=True) + transl_tags = translate_tags_to_keep(name_tags=True, osmium=False, use_repo=True) self.assertEqual(names_tags_win, transl_tags) diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 012b8e3e..6852d592 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -26,16 +26,10 @@ class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" -def translate_tags_to_keep(name_tags=False, sys_platform='', use_repo=False): +def translate_tags_to_keep(name_tags=False, osmium=True, use_repo=False): """ translates the given tags to format of the operating system. """ - - if sys_platform == "Windows": - separator = ' =' - else: - separator = ', ' - tags_modif = [] # read tags-to-keep .json from user-dir in favor of python installation @@ -61,21 +55,22 @@ def translate_tags_to_keep(name_tags=False, sys_platform='', use_repo=False): universal_tags = tags_from_json['NAME_TAGS_TO_KEEP_UNIVERSAL'] for tag, value in universal_tags.items(): - to_append = transl_tag_value(sys_platform, separator, tag, value) + to_append = transl_tag_value(osmium, tag, value) tags_modif.append(to_append) - if sys_platform == "Windows": + if not osmium: tags_modif = ' '.join(tags_modif) return tags_modif -def transl_tag_value(sys_platform, separator, tag, value): +def transl_tag_value(osmium, tag, value): """ translates one tag with value(s) to a "common" format """ if isinstance(value, list): + separator = ', ' if osmium else ' =' for iteration, sing_val in enumerate(value): if iteration == 0: to_append = f'{tag}={sing_val}' @@ -84,10 +79,7 @@ def transl_tag_value(sys_platform, separator, tag, value): elif value: to_append = f'{tag}={value}' else: - if sys_platform == "Windows": - to_append = f'{tag}=' - else: - to_append = tag + to_append = tag if osmium else f'{tag}=' return to_append diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 2dd0a886..17f9219a 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -132,10 +132,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st '+ Filtering unwanted map objects out of map of %s', key) cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append( - '--keep="' + translate_tags_to_keep(sys_platform=platform.system()) + '"') - cmd.append('--keep-tags="all type= layer= ' + - translate_tags_to_keep(sys_platform=platform.system()) + '"') + cmd.append('--keep="' + translate_tags_to_keep(osmium=False) + '"') + cmd.append('--keep-tags="all type= layer= ' + translate_tags_to_keep(osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_win) run_subprocess_and_log_output( @@ -143,12 +141,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append( - '--keep="' + translate_tags_to_keep( - name_tags=True, sys_platform=platform.system()) + '"') - cmd.append('--keep-tags="all type= name= layer= ' + - translate_tags_to_keep( - name_tags=True, sys_platform=platform.system()) + '"') + cmd.append('--keep="' + translate_tags_to_keep(name_tags=True, osmium=False) + '"') + cmd.append('--keep-tags="all type= name= layer= ' + translate_tags_to_keep(name_tags=True, osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_names_win) run_subprocess_and_log_output( @@ -175,8 +169,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep( - sys_platform=platform.system())) + cmd.extend(translate_tags_to_keep()) cmd.extend(['-o', out_file_pbf_filtered_mac]) cmd.append('--overwrite') @@ -185,8 +178,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep( - name_tags=True, sys_platform=platform.system())) + cmd.extend(translate_tags_to_keep(name_tags=True)) cmd.extend(['-o', out_file_pbf_filtered_names_mac]) cmd.append('--overwrite') @@ -724,8 +716,8 @@ def write_country_config_file(self, country): configuration = { "version_last_run": VERSION, "changed_ts_map_last_run": get_timestamp_last_changed(self.o_osm_data.border_countries[country]['map_file']), - "tags_last_run": translate_tags_to_keep(sys_platform=platform.system()), - "name_tags_last_run": translate_tags_to_keep(name_tags=True, sys_platform=platform.system()) + "tags_last_run": translate_tags_to_keep(), + "name_tags_last_run": translate_tags_to_keep(name_tags=True) } write_json_file_generic(os.path.join( @@ -740,8 +732,8 @@ def tags_are_identical_to_last_run(self, country): try: country_config = read_json_file_country_config(os.path.join( USER_OUTPUT_DIR, country, ".config.json")) - if not country_config["tags_last_run"] == translate_tags_to_keep(sys_platform=platform.system()) \ - or not country_config["name_tags_last_run"] == translate_tags_to_keep(name_tags=True, sys_platform=platform.system()): + if not country_config["tags_last_run"] == translate_tags_to_keep() \ + or not country_config["name_tags_last_run"] == translate_tags_to_keep(name_tags=True): tags_are_identical = False except (FileNotFoundError, KeyError): tags_are_identical = False From be53260e17438478c0d2e3b8f629fba1d2265fb0 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 11:16:53 +0200 Subject: [PATCH 08/17] get rid of unused issue_message argument It wasn't used consistently anyway. --- wahoomc/input.py | 11 ++++------- wahoomc/main.py | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/wahoomc/input.py b/wahoomc/input.py index 25bc55bc..3b3293cb 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -201,7 +201,7 @@ def __init__(self): self.verbose = False - def is_required_input_given_or_exit(self, issue_message): + def is_required_input_given_or_exit(self): """ check, if the minimal required arguments is given: - country @@ -210,11 +210,8 @@ def is_required_input_given_or_exit(self, issue_message): If not, depending on the import parameter, the """ if (self.country in ('None', '') and self.xy_coordinates in ('None', '')): - if issue_message: - sys.exit("Nothing to do. Start with -h or --help to see command line options." - "Or in the GUI select a country to create maps for.") - else: - sys.exit() + sys.exit("Nothing to do. Start with -h or --help to see command line options." + "Or in the GUI select a country to create maps for.") elif self.country and self.xy_coordinates: sys.exit( "Country and X/Y coordinates are given. Only one of both is allowed!") @@ -250,7 +247,7 @@ def start_gui(self): # start GUI self.mainloop() - self.o_input_data.is_required_input_given_or_exit(issue_message=True) + self.o_input_data.is_required_input_given_or_exit() return self.o_input_data def build_gui(self): diff --git a/wahoomc/main.py b/wahoomc/main.py index 79ce9a37..8a8f4584 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -54,7 +54,7 @@ def run(run_level): copy_jsons_from_repo_to_user('.', 'tags-to-keep.json') else: # Is there something to do? - o_input_data.is_required_input_given_or_exit(issue_message=True) + o_input_data.is_required_input_given_or_exit() if o_input_data.contour: check_installation_of_programs_credentials_for_contour_lines() From 3fd32170ff8b0e7d6257664397fa2a839f61dc8f Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 11:19:05 +0200 Subject: [PATCH 09/17] clean up is_required_input_given_or_exit() No need for "else" if an error case sys.exit()s. --- wahoomc/input.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/wahoomc/input.py b/wahoomc/input.py index 3b3293cb..8a2aba36 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -201,7 +201,8 @@ def __init__(self): self.verbose = False - def is_required_input_given_or_exit(self): + + def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches """ check, if the minimal required arguments is given: - country @@ -212,20 +213,18 @@ def is_required_input_given_or_exit(self): if (self.country in ('None', '') and self.xy_coordinates in ('None', '')): sys.exit("Nothing to do. Start with -h or --help to see command line options." "Or in the GUI select a country to create maps for.") - elif self.country and self.xy_coordinates: - sys.exit( - "Country and X/Y coordinates are given. Only one of both is allowed!") - elif self.country: + + if self.country and self.xy_coordinates: + sys.exit("Country and X/Y coordinates are given. Only one of both is allowed!") + + if self.country: # countries = try: CountryGeofabrik.split_input_to_list(self.country) except CountyIsNoGeofabrikCountry as exception: sys.exit(exception) - # if we made it until here, sys.exit() was not called and therefore all countries OK ;-) - return True - else: - return True + return True class GuiInput(tk.Tk): From 102df91e3ee4145f0061d27a12ea825435831783 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 11:26:54 +0200 Subject: [PATCH 10/17] allow setting an arbitrary --tag_wahoo_xml file Don't just look in the user config directory and the bundled resources/ folder, allow passing any filename to --tag_wahoo_xml. Order of preference: - filesystem, either relative or absolute - relative to the user config directory - relative to the bundled resources directory --- tests/test_constants.py | 28 +--------------------------- wahoomc/constants_functions.py | 19 ------------------- wahoomc/input.py | 13 ++++++++++++- wahoomc/osm_maps_functions.py | 14 +++----------- 4 files changed, 16 insertions(+), 58 deletions(-) diff --git a/tests/test_constants.py b/tests/test_constants.py index 1c9b74d7..05660015 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -1,14 +1,11 @@ """ tests for the downloader file """ -import os import unittest import mock -from wahoomc.constants_functions import translate_tags_to_keep, \ - get_tag_wahoo_xml_path, TagWahooXmlNotFoundError -from wahoomc.constants import RESOURCES_DIR +from wahoomc.constants_functions import translate_tags_to_keep tags_universal_simple = {"TAGS_TO_KEEP_UNIVERSAL": { @@ -124,28 +121,5 @@ def test_translate_name_tags_to_keep_full_win(self): self.assertEqual(names_tags_win, transl_tags) -class TestTagWahooXML(unittest.TestCase): - """ - tests for tag-wahoo xml file - """ - - def test_not_existing_tag_wahoo_xml(self): - """ - check if a non-existing tag-wahoo xml file issues an exception - """ - with self.assertRaises(TagWahooXmlNotFoundError): - get_tag_wahoo_xml_path("not_existing.xml") - - def test_existing_tag_wahoo_xml(self): - """ - check if the correct path of an existing tag-wahoo xml file is returned - """ - - expected_path = os.path.join( - RESOURCES_DIR, "tag_wahoo_adjusted", "tag-wahoo-poi.xml") - self.assertEqual(get_tag_wahoo_xml_path( - "tag-wahoo-poi.xml"), expected_path) - - if __name__ == '__main__': unittest.main() diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 6852d592..3160fd15 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -18,10 +18,6 @@ log = logging.getLogger('main-logger') -class TagWahooXmlNotFoundError(Exception): - """Raised when the specified tag-wahoo xml file does not exist""" - - class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" @@ -104,21 +100,6 @@ def get_tooling_win_path(path_in_tooling_win, in_user_dir=False): # all other "toolings": concatenate with win tooling dir return os.path.join(tooling_dir, path_in_tooling_win) - -def get_tag_wahoo_xml_path(tag_wahoo_xml): - """ - return path to tag-wahoo xml file if the file exists - - from the user directory "USER_WAHOO_MC/_config/tag_wahoo_adjusted/tag_wahoo_xml" - - 2ndly from the PyPI installation: "RESOURCES_DIR/tag_wahoo_adjusted/tag_wahoo_xml" - """ - - for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", tag_wahoo_xml): - if os.path.exists(path): - return path - - raise TagWahooXmlNotFoundError - - def get_absolute_dir_user_or_repo(folder, file=''): """ return the absolute path to the folder (and file) in this priorization diff --git a/wahoomc/input.py b/wahoomc/input.py index 8a2aba36..6272ba23 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -5,6 +5,7 @@ # import official python packages import argparse +import os import sys from platform import uname @@ -13,6 +14,7 @@ from tkinter import ttk # import custom python packages +from wahoomc.constants_functions import get_absolute_dir_user_or_repo from wahoomc.geofabrik_json import GeofabrikJson from wahoomc.geofabrik_json import CountyIsNoGeofabrikCountry from wahoomc.geofabrik import CountryGeofabrik @@ -78,7 +80,7 @@ def process_call_of_the_tool(): # Save uncompressed maps for Cruiser if True options_args.add_argument('-c', '--cruiser', action='store_true', help="save uncompressed maps for Cruiser") - # specify the file with tags to keep in the output // file needs to be in wahoo_mc/resources/tag_wahoo_adjusted + # specify the file with tags to keep in the output options_args.add_argument('-tag', '--tag_wahoo_xml', default=InputData().tag_wahoo_xml, help="file with tags to keep in the output") # zip the country (and country-maps) folder @@ -224,6 +226,15 @@ def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches except CountyIsNoGeofabrikCountry as exception: sys.exit(exception) + if not os.path.exists(self.tag_wahoo_xml): + for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", self.tag_wahoo_xml): + if os.path.exists(path): + self.tag_wahoo_xml = path + break + + if not os.path.exists(self.tag_wahoo_xml): + sys.exit(f'The tag-wahoo xml file was not found: \"{self.tag_wahoo_xml}\"') + return True diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 17f9219a..7b0be2ee 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -16,8 +16,7 @@ # import custom python packages from wahoomc.file_directory_functions import read_json_file_country_config, create_empty_directories, write_json_file_generic -from wahoomc.constants_functions import translate_tags_to_keep, \ - get_tooling_win_path, get_tag_wahoo_xml_path, TagWahooXmlNotFoundError +from wahoomc.constants_functions import translate_tags_to_keep, get_tooling_win_path from wahoomc.setup_functions import read_earthexplorer_credentials @@ -528,7 +527,7 @@ def sort_osm_files(self, tile): log.debug('+ Sorting land* osm files: OK') - def create_map_files(self, save_cruiser, tag_wahoo_xml, hdd_mode): + def create_map_files(self, save_cruiser, tag_conf_file, hdd_mode): """ Creating .map files """ @@ -569,14 +568,7 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml, hdd_mode): cmd.append(f'threads={threads}') if hdd_mode: cmd.append('type=hd') - # add path to tag-wahoo xml file - try: - cmd.append( - f'tag-conf-file={get_tag_wahoo_xml_path(tag_wahoo_xml)}') - except TagWahooXmlNotFoundError: - log.error( - 'The tag-wahoo xml file was not found: ˚%s˚. Does the file exist and is your input correct?', tag_wahoo_xml) - sys.exit() + cmd.append(f'tag-conf-file={tag_conf_file}') run_subprocess_and_log_output( cmd, f'Error in creating map file via Osmosis with tile: {tile["x"]},{tile["y"]}. mapwriter plugin installed?') From 3c0b86e47128378fb0346476b05a9f78b25cbb9b Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 11:57:17 +0200 Subject: [PATCH 11/17] log the used map-writer tag-conf file --- wahoomc/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wahoomc/main.py b/wahoomc/main.py index 8a8f4584..64646a49 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -18,6 +18,8 @@ from wahoomc.osm_maps_functions import OsmMaps from wahoomc.osm_data import CountryOsmData, XYOsmData +log = logging.getLogger('main-logger') + # logging used in the terminal output: # # means top-level command # ! means error @@ -47,7 +49,7 @@ def run(run_level): o_input_data = process_call_of_the_tool() if o_input_data.verbose: - logging.getLogger().setLevel(logging.DEBUG) + log.setLevel(logging.DEBUG) if run_level == 'init': copy_jsons_from_repo_to_user('tag_wahoo_adjusted') @@ -59,6 +61,9 @@ def run(run_level): if o_input_data.contour: check_installation_of_programs_credentials_for_contour_lines() + log.info('# Used configuration') + log.info('+ map-writer tag-conf file: %s', o_input_data.tag_wahoo_xml) + if o_input_data.country: o_osm_data = CountryOsmData(o_input_data) else: From ec5967e3bb3c92444e065667f2b12c98163b0079 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 12:36:12 +0200 Subject: [PATCH 12/17] add --tags_to_keep to set an arbitrary tags-to-keep.json file Use the same logic as with --tag_wahoo_xml --- tests/test_constants.py | 21 +++++++++++++-------- tests/test_osm_maps.py | 18 +++++++++++++----- wahoomc/constants_functions.py | 17 +++-------------- wahoomc/input.py | 14 ++++++++++++++ wahoomc/main.py | 3 ++- wahoomc/osm_maps_functions.py | 23 ++++++++++++----------- 6 files changed, 57 insertions(+), 39 deletions(-) diff --git a/tests/test_constants.py b/tests/test_constants.py index 05660015..cc158355 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -1,10 +1,12 @@ """ tests for the downloader file """ +import os import unittest import mock +from wahoomc.constants import RESOURCES_DIR from wahoomc.constants_functions import translate_tags_to_keep @@ -27,6 +29,9 @@ class TestTranslateTags(unittest.TestCase): tests for translating tags-constants between the universal format and OS-specific formats """ + def setUp(self): + self.tags_to_keep = os.path.join(RESOURCES_DIR, 'tags-to-keep.json') + @ mock.patch("wahoomc.file_directory_functions.json.load") @ mock.patch("wahoomc.open") def test_translate_tags_to_keep_simple_macos(self, mock_open, mock_json_load): # pylint: disable=unused-argument @@ -36,7 +41,7 @@ def test_translate_tags_to_keep_simple_macos(self, mock_open, mock_json_load): tags = ['access', 'area=yes'] mock_json_load.return_value = tags_universal_simple - transl_tags = translate_tags_to_keep() + transl_tags = translate_tags_to_keep('nonexistant.json') self.assertEqual(tags, transl_tags) @ mock.patch("wahoomc.file_directory_functions.json.load") @@ -48,7 +53,7 @@ def test_translate_tags_to_keep_simple_win(self, mock_open, mock_json_load): # tags_win = 'access= area=yes' mock_json_load.return_value = tags_universal_simple - transl_tags = translate_tags_to_keep(osmium=False) + transl_tags = translate_tags_to_keep('nonexistant.json', osmium=False) self.assertEqual(tags_win, transl_tags) @ mock.patch("wahoomc.file_directory_functions.json.load") @@ -61,7 +66,7 @@ def test_translate_tags_to_keep_adv_macos(self, mock_open, mock_json_load): # p 'bridge', 'foot=ft_yes, foot_designated'] mock_json_load.return_value = tags_universal_adv - transl_tags = translate_tags_to_keep() + transl_tags = translate_tags_to_keep('nonexistant.json') self.assertEqual(tags, transl_tags) @ mock.patch("wahoomc.file_directory_functions.json.load") @@ -73,7 +78,7 @@ def test_translate_tags_to_keep_adv_win(self, mock_open, mock_json_load): # pyl tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated' mock_json_load.return_value = tags_universal_adv - transl_tags = translate_tags_to_keep(osmium=False) + transl_tags = translate_tags_to_keep('nonexistant.json', osmium=False) self.assertEqual(tags_win, transl_tags) def test_translate_tags_to_keep_full_macos(self): @@ -88,7 +93,7 @@ def test_translate_tags_to_keep_full_macos(self): 'leisure=park, nature_reserve', 'railway=rail, tram, station, stop', 'surface', 'tracktype', 'tunnel', 'waterway=canal, drain, river, riverbank, stream', 'wood=deciduous', 'tourism=alpine_hut'] - transl_tags = translate_tags_to_keep(use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep) self.assertEqual(tags, transl_tags) def test_translate_tags_to_keep_full_win(self): @@ -97,7 +102,7 @@ def test_translate_tags_to_keep_full_win(self): """ tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated amenity=fuel =cafe =drinking_water =shelter shop=bakery =bicycle highway=abandoned =bus_guideway =disused =bridleway =byway =construction =cycleway =footway =living_street =motorway =motorway_link =path =pedestrian =primary =primary_link =residential =road =secondary =secondary_link =service =steps =tertiary =tertiary_link =track =trunk =trunk_link =unclassified natural=coastline =nosea =sea =beach =land =scrub =water =wetland =wood landuse=forest =commercial =industrial =residential =retail leisure=park =nature_reserve railway=rail =tram =station =stop surface= tracktype= tunnel= waterway=canal =drain =river =riverbank =stream wood=deciduous tourism=alpine_hut' - transl_tags = translate_tags_to_keep(osmium=False, use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, osmium=False) self.assertEqual(tags_win, transl_tags) def test_translate_name_tags_to_keep_full_macos(self): @@ -107,7 +112,7 @@ def test_translate_name_tags_to_keep_full_macos(self): names_tags = ['admin_level=2', 'area=yes', 'mountain_pass', 'natural', 'place=city, hamlet, island, isolated_dwelling, islet, locality, suburb, town, village, country'] - transl_tags = translate_tags_to_keep(name_tags=True, use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, name_tags=True) self.assertEqual(names_tags, transl_tags) def test_translate_name_tags_to_keep_full_win(self): @@ -117,7 +122,7 @@ def test_translate_name_tags_to_keep_full_win(self): names_tags_win = 'admin_level=2 area=yes mountain_pass= natural= place=city =hamlet =island =isolated_dwelling =islet =locality =suburb =town =village =country' - transl_tags = translate_tags_to_keep(name_tags=True, osmium=False, use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) self.assertEqual(names_tags_win, transl_tags) diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index 73008f23..485668be 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -14,7 +14,15 @@ from wahoomc import constants -class TestOsmMapsCalculation(unittest.TestCase): +class TestOsmMaps(unittest.TestCase): + """ + base test class + """ + + def setUp(self): + self.tags_to_keep = os.path.join(constants.RESOURCES_DIR, 'tags-to-keep.json') + +class TestOsmMapsCalculation(TestOsmMaps): """ tests for the OSM maps file """ @@ -164,7 +172,7 @@ def process_and_check_border_countries(self, inp_val, calc_border_c, exp_result, self.assertEqual(result, exp_result) -class TestOSMMapsInput(unittest.TestCase): +class TestOSMMapsInput(TestOsmMaps): """ tests for input of OsmData """ @@ -185,7 +193,7 @@ def test_folder_name_many_countries(self): """ o_osm_data = self.get_osm_data_instance('albania,alps,andorra,austria,azores,belarus,belgium,bosnia-herzegovina,britain-and-ireland,bulgaria,croatia,cyprus,czech-republic,dach,denmark,estonia,faroe-islands,finland,france,georgia,germany,great-britain,greece,guernsey-jersey,hungary,iceland,ireland-and-northern-ireland,isle-of-man,italy,kosovo,latvia,liechtenstein,lithuania,luxembourg,macedonia,malta,moldova,monaco,montenegro,netherlands,norway,poland,portugal,romania,serbia,slovakia,slovenia,spain,sweden,switzerland,turkey,ukraine') - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep) folder_name = o_osm_maps.calculate_folder_name('.map.lzma') folder_name_maps = o_osm_maps.calculate_folder_name('.map') @@ -216,7 +224,7 @@ def get_osm_data_instance(self, country_input): return o_osm_data -class TestConfigFile(unittest.TestCase): +class TestConfigFile(TestOsmMaps): """ tests for the config .json file in the "wahooMapsCreatorData/_tiles/{country}" directory """ @@ -239,7 +247,7 @@ def test_version_and_tags_of_country_config_file(self): # download files marked for download to fill up map_file per country to write to config o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep) o_osm_maps.write_country_config_file(o_input_data.country) diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 3160fd15..53e2d59d 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -22,25 +22,14 @@ class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" -def translate_tags_to_keep(name_tags=False, osmium=True, use_repo=False): +def translate_tags_to_keep(tags_to_keep, name_tags=False, osmium=True): """ translates the given tags to format of the operating system. """ tags_modif = [] - # read tags-to-keep .json from user-dir in favor of python installation - # evaluate path first: user-dir in favor of PyPI installation - if not use_repo: - for path in get_absolute_dir_user_or_repo('', file='tags-to-keep.json'): - if os.path.exists(path): - break - # force using file from repo - used in unittests for equal output - else: - path = get_absolute_dir_user_or_repo( - '', file='tags-to-keep.json')[1] - - # read the tags from the evaluated path above - tags_from_json = read_json_file_generic(path) + # read the tags from the passed tags_to_keep path + tags_from_json = read_json_file_generic(tags_to_keep) if not tags_from_json: raise TagsToKeepNotFoundError diff --git a/wahoomc/input.py b/wahoomc/input.py index 6272ba23..6afb7495 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -80,6 +80,9 @@ def process_call_of_the_tool(): # Save uncompressed maps for Cruiser if True options_args.add_argument('-c', '--cruiser', action='store_true', help="save uncompressed maps for Cruiser") + # specify the file with tags to keep when filtering + options_args.add_argument('--tags_to_keep', default=InputData().tags_to_keep, + help="file with tags to keep when filtering") # specify the file with tags to keep in the output options_args.add_argument('-tag', '--tag_wahoo_xml', default=InputData().tag_wahoo_xml, help="file with tags to keep in the output") @@ -116,6 +119,7 @@ def process_call_of_the_tool(): o_input_data.force_download = args.forcedownload o_input_data.force_processing = args.forceprocessing + o_input_data.tags_to_keep = args.tags_to_keep o_input_data.tag_wahoo_xml = args.tag_wahoo_xml o_input_data.save_cruiser = args.cruiser o_input_data.zip_folder = args.zip @@ -196,6 +200,7 @@ def __init__(self): self.contour = False self.use_srtm1 = False + self.tags_to_keep = "tags-to-keep.json" self.tag_wahoo_xml = "tag-wahoo-poi.xml" self.zip_folder = False self.save_cruiser = False @@ -226,6 +231,15 @@ def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches except CountyIsNoGeofabrikCountry as exception: sys.exit(exception) + if not os.path.exists(self.tags_to_keep): + for path in get_absolute_dir_user_or_repo("", self.tags_to_keep): + if os.path.exists(path): + self.tags_to_keep = path + break + + if not os.path.exists(self.tags_to_keep): + sys.exit(f'The tags-to-keep json file was not found: \"{self.tags_to_keep}\"') + if not os.path.exists(self.tag_wahoo_xml): for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", self.tag_wahoo_xml): if os.path.exists(path): diff --git a/wahoomc/main.py b/wahoomc/main.py index 64646a49..aaceb0d5 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -62,6 +62,7 @@ def run(run_level): check_installation_of_programs_credentials_for_contour_lines() log.info('# Used configuration') + log.info('+ tags-to-keep file: %s', o_input_data.tags_to_keep) log.info('+ map-writer tag-conf file: %s', o_input_data.tag_wahoo_xml) if o_input_data.country: @@ -77,7 +78,7 @@ def run(run_level): # Download files marked for download o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, o_input_data.tags_to_keep) # Filter tags from country osm.pbf files' o_osm_maps.filter_tags_from_country_osm_pbf_files() diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 7b0be2ee..18044bba 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -75,8 +75,9 @@ class OsmMaps: # Number of workers for the Osmosis read binary fast function workers = '1' - def __init__(self, o_osm_data): + def __init__(self, o_osm_data, tags_to_keep): self.o_osm_data = o_osm_data + self.tags_to_keep = tags_to_keep self.osmconvert_path = get_tooling_win_path('osmconvert') create_empty_directories( @@ -131,8 +132,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st '+ Filtering unwanted map objects out of map of %s', key) cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append('--keep="' + translate_tags_to_keep(osmium=False) + '"') - cmd.append('--keep-tags="all type= layer= ' + translate_tags_to_keep(osmium=False) + '"') + cmd.append('--keep="' + translate_tags_to_keep(self.tags_to_keep, osmium=False) + '"') + cmd.append('--keep-tags="all type= layer= ' + translate_tags_to_keep(self.tags_to_keep, osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_win) run_subprocess_and_log_output( @@ -140,8 +141,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append('--keep="' + translate_tags_to_keep(name_tags=True, osmium=False) + '"') - cmd.append('--keep-tags="all type= name= layer= ' + translate_tags_to_keep(name_tags=True, osmium=False) + '"') + cmd.append('--keep="' + translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) + '"') + cmd.append('--keep-tags="all type= name= layer= ' + translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_names_win) run_subprocess_and_log_output( @@ -168,7 +169,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep()) + cmd.extend(translate_tags_to_keep(self.tags_to_keep)) cmd.extend(['-o', out_file_pbf_filtered_mac]) cmd.append('--overwrite') @@ -177,7 +178,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep(name_tags=True)) + cmd.extend(translate_tags_to_keep(self.tags_to_keep, name_tags=True)) cmd.extend(['-o', out_file_pbf_filtered_names_mac]) cmd.append('--overwrite') @@ -708,8 +709,8 @@ def write_country_config_file(self, country): configuration = { "version_last_run": VERSION, "changed_ts_map_last_run": get_timestamp_last_changed(self.o_osm_data.border_countries[country]['map_file']), - "tags_last_run": translate_tags_to_keep(), - "name_tags_last_run": translate_tags_to_keep(name_tags=True) + "tags_last_run": translate_tags_to_keep(self.tags_to_keep), + "name_tags_last_run": translate_tags_to_keep(self.tags_to_keep, name_tags=True) } write_json_file_generic(os.path.join( @@ -724,8 +725,8 @@ def tags_are_identical_to_last_run(self, country): try: country_config = read_json_file_country_config(os.path.join( USER_OUTPUT_DIR, country, ".config.json")) - if not country_config["tags_last_run"] == translate_tags_to_keep() \ - or not country_config["name_tags_last_run"] == translate_tags_to_keep(name_tags=True): + if not country_config["tags_last_run"] == translate_tags_to_keep(self.tags_to_keep) \ + or not country_config["name_tags_last_run"] == translate_tags_to_keep(self.tags_to_keep, name_tags=True): tags_are_identical = False except (FileNotFoundError, KeyError): tags_are_identical = False From 34ccfdb042c88085ac00acec470ed6c9316af82c Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 22:44:10 +0200 Subject: [PATCH 13/17] copy the current tags-to-keep file to tests/ And test that instead. There's no need to adapt the test everytime the file gets changed. --- tests/resources/tags-to-keep.json | 109 ++++++++++++++++++++++++++++++ tests/test_constants.py | 3 +- 2 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 tests/resources/tags-to-keep.json diff --git a/tests/resources/tags-to-keep.json b/tests/resources/tags-to-keep.json new file mode 100644 index 00000000..b306b27c --- /dev/null +++ b/tests/resources/tags-to-keep.json @@ -0,0 +1,109 @@ +{ + "TAGS_TO_KEEP_UNIVERSAL": { + "access": "", + "area": "yes", + "bicycle": "", + "bridge": "", + "foot": [ + "ft_yes", + "foot_designated" + ], + "amenity": [ + "fuel", + "cafe", + "drinking_water", + "shelter" + ], + "shop": [ + "bakery", + "bicycle" + ], + "highway": [ + "abandoned", + "bus_guideway", + "disused", + "bridleway", + "byway", + "construction", + "cycleway", + "footway", + "living_street", + "motorway", + "motorway_link", + "path", + "pedestrian", + "primary", + "primary_link", + "residential", + "road", + "secondary", + "secondary_link", + "service", + "steps", + "tertiary", + "tertiary_link", + "track", + "trunk", + "trunk_link", + "unclassified" + ], + "natural": [ + "coastline", + "nosea", + "sea", + "beach", + "land", + "scrub", + "water", + "wetland", + "wood" + ], + "landuse": [ + "forest", + "commercial", + "industrial", + "residential", + "retail" + ], + "leisure": [ + "park", + "nature_reserve" + ], + "railway": [ + "rail", + "tram", + "station", + "stop" + ], + "surface": "", + "tracktype": "", + "tunnel": "", + "waterway": [ + "canal", + "drain", + "river", + "riverbank", + "stream" + ], + "wood": "deciduous", + "tourism": "alpine_hut" + }, + "NAME_TAGS_TO_KEEP_UNIVERSAL": { + "admin_level": "2", + "area": "yes", + "mountain_pass": "", + "natural": "", + "place": [ + "city", + "hamlet", + "island", + "isolated_dwelling", + "islet", + "locality", + "suburb", + "town", + "village", + "country" + ] + } +} \ No newline at end of file diff --git a/tests/test_constants.py b/tests/test_constants.py index cc158355..d5e23eeb 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -6,7 +6,6 @@ import mock -from wahoomc.constants import RESOURCES_DIR from wahoomc.constants_functions import translate_tags_to_keep @@ -30,7 +29,7 @@ class TestTranslateTags(unittest.TestCase): """ def setUp(self): - self.tags_to_keep = os.path.join(RESOURCES_DIR, 'tags-to-keep.json') + self.tags_to_keep = os.path.join(os.path.dirname(__file__), 'resources', 'tags-to-keep.json') @ mock.patch("wahoomc.file_directory_functions.json.load") @ mock.patch("wahoomc.open") From 35ca814ec84317f3245ea5398b3b3b555aca6d7f Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 22:45:31 +0200 Subject: [PATCH 14/17] get rid of mock The functionality is already tested with real files, there's no need to bend over backwards and hack around with mock, which doesn't work in its current state anyway. --- .github/workflows/tests.yml | 1 - conda_env/gdal-dev.yml | 1 - tests/test_constants.py | 64 ------------------------------------- 3 files changed, 66 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cea4432c..2b847eb7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,6 @@ jobs: run: | python -m pip install --upgrade pip pip install pylint==3.3.* - pip install mock pip install requests==2.28.* - name: Analysing the code with pylint run: | diff --git a/conda_env/gdal-dev.yml b/conda_env/gdal-dev.yml index 08333e8b..e30d60cc 100644 --- a/conda_env/gdal-dev.yml +++ b/conda_env/gdal-dev.yml @@ -13,7 +13,6 @@ dependencies: - lxml=4.9.* - matplotlib=3.4.3 - autopep8=2.0.* - - mock - twine - pip - vulture diff --git a/tests/test_constants.py b/tests/test_constants.py index d5e23eeb..0039364c 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -3,26 +3,11 @@ """ import os import unittest -import mock from wahoomc.constants_functions import translate_tags_to_keep -tags_universal_simple = {"TAGS_TO_KEEP_UNIVERSAL": { - 'access': '', - 'area': 'yes' -}} - -tags_universal_adv = {"TAGS_TO_KEEP_UNIVERSAL": { - 'access': '', - 'area': 'yes', - 'bicycle': '', - 'bridge': '', - 'foot': ['ft_yes', 'foot_designated'] -}} - - class TestTranslateTags(unittest.TestCase): """ tests for translating tags-constants between the universal format and OS-specific formats @@ -31,55 +16,6 @@ class TestTranslateTags(unittest.TestCase): def setUp(self): self.tags_to_keep = os.path.join(os.path.dirname(__file__), 'resources', 'tags-to-keep.json') - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_simple_macos(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to macOS - """ - tags = ['access', 'area=yes'] - mock_json_load.return_value = tags_universal_simple - - transl_tags = translate_tags_to_keep('nonexistant.json') - self.assertEqual(tags, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_simple_win(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to Windows - """ - tags_win = 'access= area=yes' - mock_json_load.return_value = tags_universal_simple - - transl_tags = translate_tags_to_keep('nonexistant.json', osmium=False) - self.assertEqual(tags_win, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_adv_macos(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to macOS - """ - tags = ['access', 'area=yes', 'bicycle', - 'bridge', 'foot=ft_yes, foot_designated'] - mock_json_load.return_value = tags_universal_adv - - transl_tags = translate_tags_to_keep('nonexistant.json') - self.assertEqual(tags, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_adv_win(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to Windows - """ - tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated' - mock_json_load.return_value = tags_universal_adv - - transl_tags = translate_tags_to_keep('nonexistant.json', osmium=False) - self.assertEqual(tags_win, transl_tags) - def test_translate_tags_to_keep_full_macos(self): """ Test translating tags to keep from universal format to macOS // all "tags to keep" From dbc0ad9d8f2bf8f6b4480f85dbd44be2bbb5aef4 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 12:57:27 +0200 Subject: [PATCH 15/17] add --tag_transform to set an arbitrary tag-transform file Use the same logic as with --tag_wahoo_xml --- tests/test_osm_maps.py | 5 +++-- wahoomc/input.py | 14 ++++++++++++++ wahoomc/main.py | 3 ++- wahoomc/osm_maps_functions.py | 6 +++--- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index 485668be..962fa005 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -21,6 +21,7 @@ class TestOsmMaps(unittest.TestCase): def setUp(self): self.tags_to_keep = os.path.join(constants.RESOURCES_DIR, 'tags-to-keep.json') + self.tag_transform = os.path.join(constants.RESOURCES_DIR, 'tunnel-transform.xml') class TestOsmMapsCalculation(TestOsmMaps): """ @@ -193,7 +194,7 @@ def test_folder_name_many_countries(self): """ o_osm_data = self.get_osm_data_instance('albania,alps,andorra,austria,azores,belarus,belgium,bosnia-herzegovina,britain-and-ireland,bulgaria,croatia,cyprus,czech-republic,dach,denmark,estonia,faroe-islands,finland,france,georgia,germany,great-britain,greece,guernsey-jersey,hungary,iceland,ireland-and-northern-ireland,isle-of-man,italy,kosovo,latvia,liechtenstein,lithuania,luxembourg,macedonia,malta,moldova,monaco,montenegro,netherlands,norway,poland,portugal,romania,serbia,slovakia,slovenia,spain,sweden,switzerland,turkey,ukraine') - o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep, self.tag_transform) folder_name = o_osm_maps.calculate_folder_name('.map.lzma') folder_name_maps = o_osm_maps.calculate_folder_name('.map') @@ -247,7 +248,7 @@ def test_version_and_tags_of_country_config_file(self): # download files marked for download to fill up map_file per country to write to config o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep, self.tag_transform) o_osm_maps.write_country_config_file(o_input_data.country) diff --git a/wahoomc/input.py b/wahoomc/input.py index 6afb7495..450be9a3 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -83,6 +83,9 @@ def process_call_of_the_tool(): # specify the file with tags to keep when filtering options_args.add_argument('--tags_to_keep', default=InputData().tags_to_keep, help="file with tags to keep when filtering") + # specify the file with tag-transform rules + options_args.add_argument('--tag_transform', default=InputData().tag_transform, + help="file with tag-transform rules") # specify the file with tags to keep in the output options_args.add_argument('-tag', '--tag_wahoo_xml', default=InputData().tag_wahoo_xml, help="file with tags to keep in the output") @@ -120,6 +123,7 @@ def process_call_of_the_tool(): o_input_data.force_processing = args.forceprocessing o_input_data.tags_to_keep = args.tags_to_keep + o_input_data.tag_transform = args.tag_transform o_input_data.tag_wahoo_xml = args.tag_wahoo_xml o_input_data.save_cruiser = args.cruiser o_input_data.zip_folder = args.zip @@ -201,6 +205,7 @@ def __init__(self): self.use_srtm1 = False self.tags_to_keep = "tags-to-keep.json" + self.tag_transform = "tunnel-transform.xml" self.tag_wahoo_xml = "tag-wahoo-poi.xml" self.zip_folder = False self.save_cruiser = False @@ -240,6 +245,15 @@ def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches if not os.path.exists(self.tags_to_keep): sys.exit(f'The tags-to-keep json file was not found: \"{self.tags_to_keep}\"') + if not os.path.exists(self.tag_transform): + for path in get_absolute_dir_user_or_repo("", self.tag_transform): + if os.path.exists(path): + self.tag_transform = path + break + + if not os.path.exists(self.tag_transform): + sys.exit(f'The tag-transform xml file was not found: \"{self.tag_transform}\"') + if not os.path.exists(self.tag_wahoo_xml): for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", self.tag_wahoo_xml): if os.path.exists(path): diff --git a/wahoomc/main.py b/wahoomc/main.py index aaceb0d5..5fd250ef 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -63,6 +63,7 @@ def run(run_level): log.info('# Used configuration') log.info('+ tags-to-keep file: %s', o_input_data.tags_to_keep) + log.info('+ tag-transform file: %s', o_input_data.tag_transform) log.info('+ map-writer tag-conf file: %s', o_input_data.tag_wahoo_xml) if o_input_data.country: @@ -78,7 +79,7 @@ def run(run_level): # Download files marked for download o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data, o_input_data.tags_to_keep) + o_osm_maps = OsmMaps(o_osm_data, o_input_data.tags_to_keep, o_input_data.tag_transform) # Filter tags from country osm.pbf files' o_osm_maps.filter_tags_from_country_osm_pbf_files() diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 18044bba..1ee691f4 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -75,9 +75,10 @@ class OsmMaps: # Number of workers for the Osmosis read binary fast function workers = '1' - def __init__(self, o_osm_data, tags_to_keep): + def __init__(self, o_osm_data, tags_to_keep, tag_transform): self.o_osm_data = o_osm_data self.tags_to_keep = tags_to_keep + self.tag_transform = tag_transform self.osmconvert_path = get_tooling_win_path('osmconvert') create_empty_directories( @@ -488,8 +489,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto cmd.extend( ['--rx', 'file='+os.path.join(out_tile_dir, 'sea.osm'), '--s', '--m']) - cmd.extend(['--tag-transform', 'file=' + os.path.join(RESOURCES_DIR, - 'tunnel-transform.xml'), '--wb', out_file_merged, 'omitmetadata=true']) + cmd.extend(['--tag-transform', 'file=' + self.tag_transform, '--wb', out_file_merged, 'omitmetadata=true']) run_subprocess_and_log_output( cmd, f'! Error in Osmosis with tile: {tile["x"]},{tile["y"]}') From e9de2dafbe698cd274ca45a8fdc8967234096af3 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 22 May 2025 08:43:46 +0200 Subject: [PATCH 16/17] update dependency mapwriter plugin to 0.25.0 --- tests/test_downloader.py | 2 +- wahoomc/downloader.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader.py b/tests/test_downloader.py index c8b3ba55..475ef49e 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -186,7 +186,7 @@ def test_download_macos_files(self): """ if platform.system() != "Windows": path = os.path.join(str(constants.USER_DIR), '.openstreetmap', 'osmosis', - 'plugins', 'mapsforge-map-writer-0.21.0-jar-with-dependencies.jar') + 'plugins', 'mapsforge-map-writer-0.25.0-jar-with-dependencies.jar') if os.path.exists(path): os.remove(path) diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index 338f4fc4..57372b9c 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -119,8 +119,8 @@ def download_tooling(): check here for new mapwriter plugin version: https://github.com/mapsforge/mapsforge """ - map_writer_filename = 'mapsforge-map-writer-0.21.0-jar-with-dependencies.jar' - mapwriter_plugin_url = 'https://search.maven.org/remotecontent?filepath=org/mapsforge/mapsforge-map-writer/0.21.0/' + map_writer_filename + map_writer_filename = 'mapsforge-map-writer-0.25.0-jar-with-dependencies.jar' + mapwriter_plugin_url = 'https://search.maven.org/remotecontent?filepath=org/mapsforge/mapsforge-map-writer/0.25.0/' + map_writer_filename # Windows if platform.system() == "Windows": From 8c7b1f51bfaa78fd2b2ab4c316ed37d9b77b539e Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 29 May 2025 08:29:16 +0200 Subject: [PATCH 17/17] enable tag-values for the mapsforge-map-writer plugin This enables wildcard values like %f and %s. The current ones in tag-wahoo-poi.xml were ignored unit now, so remove them to keep the results unchanged. --- wahoomc/osm_maps_functions.py | 1 + wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 1ee691f4..91fb56e8 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -567,6 +567,7 @@ def create_map_files(self, save_cruiser, tag_conf_file, hdd_mode): f'bbox={tile["bottom"]:.6f},{tile["left"]:.6f},{tile["top"]:.6f},{tile["right"]:.6f}') cmd.append('zoom-interval-conf=12,0,17') cmd.append(f'threads={threads}') + cmd.append('tag-values=true') if hdd_mode: cmd.append('type=hd') cmd.append(f'tag-conf-file={tag_conf_file}') diff --git a/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml b/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml index 6f43434f..8fe06d45 100644 --- a/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml +++ b/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml @@ -171,11 +171,6 @@ - - - - -