diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 70e9ffe4..2b847eb7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,8 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pylint==2.15.* - pip install mock + pip install pylint==3.3.* pip install requests==2.28.* - name: Analysing the code with pylint run: | diff --git a/.pylintrc b/.pylintrc index a95f775e..22dd6143 100644 --- a/.pylintrc +++ b/.pylintrc @@ -17,6 +17,3 @@ disable=line-too-long, duplicate-code ; [MASTER] ; init-hook='import sys; sys.path.append("/path/to/root")' ; init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))" - -[MASTER] -init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))" diff --git a/conda_env/gdal-dev.yml b/conda_env/gdal-dev.yml index c20086aa..e30d60cc 100644 --- a/conda_env/gdal-dev.yml +++ b/conda_env/gdal-dev.yml @@ -5,7 +5,7 @@ dependencies: - python=3.10 - gdal=3.6.* - requests=2.28.* - - pylint=2.15.* + - pylint=3.3.* - geojson=2.5.* - shapely=1.8.* - osmium-tool=1.16.* @@ -13,7 +13,6 @@ dependencies: - lxml=4.9.* - matplotlib=3.4.3 - autopep8=2.0.* - - mock - twine - pip - vulture diff --git a/tests/resources/tags-to-keep.json b/tests/resources/tags-to-keep.json new file mode 100644 index 00000000..b306b27c --- /dev/null +++ b/tests/resources/tags-to-keep.json @@ -0,0 +1,109 @@ +{ + "TAGS_TO_KEEP_UNIVERSAL": { + "access": "", + "area": "yes", + "bicycle": "", + "bridge": "", + "foot": [ + "ft_yes", + "foot_designated" + ], + "amenity": [ + "fuel", + "cafe", + "drinking_water", + "shelter" + ], + "shop": [ + "bakery", + "bicycle" + ], + "highway": [ + "abandoned", + "bus_guideway", + "disused", + "bridleway", + "byway", + "construction", + "cycleway", + "footway", + "living_street", + "motorway", + "motorway_link", + "path", + "pedestrian", + "primary", + "primary_link", + "residential", + "road", + "secondary", + "secondary_link", + "service", + "steps", + "tertiary", + "tertiary_link", + "track", + "trunk", + "trunk_link", + "unclassified" + ], + "natural": [ + "coastline", + "nosea", + "sea", + "beach", + "land", + "scrub", + "water", + "wetland", + "wood" + ], + "landuse": [ + "forest", + "commercial", + "industrial", + "residential", + "retail" + ], + "leisure": [ + "park", + "nature_reserve" + ], + "railway": [ + "rail", + "tram", + "station", + "stop" + ], + "surface": "", + "tracktype": "", + "tunnel": "", + "waterway": [ + "canal", + "drain", + "river", + "riverbank", + "stream" + ], + "wood": "deciduous", + "tourism": "alpine_hut" + }, + "NAME_TAGS_TO_KEEP_UNIVERSAL": { + "admin_level": "2", + "area": "yes", + "mountain_pass": "", + "natural": "", + "place": [ + "city", + "hamlet", + "island", + "isolated_dwelling", + "islet", + "locality", + "suburb", + "town", + "village", + "country" + ] + } +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py index c2723198..aa595524 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,7 @@ tests for the python file """ import os +import sys import unittest # import custom python packages @@ -17,7 +18,7 @@ def test_top_parser_help(self): tests, if help of top parser can be called """ - result = os.system("python -m wahoomc -h") + result = os.system(sys.executable + " -m wahoomc -h") self.assertEqual(result, 0) @@ -26,7 +27,7 @@ def test_cli_help(self): tests, if CLI help can be called """ - result = os.system("python -m wahoomc cli -h") + result = os.system(sys.executable + " -m wahoomc cli -h") self.assertEqual(result, 0) @@ -35,7 +36,7 @@ def test_gui_help(self): tests, if GUI help can be called """ - result = os.system("python -m wahoomc gui -h") + result = os.system(sys.executable + " -m wahoomc gui -h") self.assertEqual(result, 0) diff --git a/tests/test_constants.py b/tests/test_constants.py index 8c6c5e82..0039364c 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -3,26 +3,9 @@ """ import os import unittest -import mock -from wahoomc.constants_functions import translate_tags_to_keep, \ - get_tag_wahoo_xml_path, TagWahooXmlNotFoundError -from wahoomc.constants import RESOURCES_DIR - - -tags_universal_simple = {"TAGS_TO_KEEP_UNIVERSAL": { - 'access': '', - 'area': 'yes' -}} - -tags_universal_adv = {"TAGS_TO_KEEP_UNIVERSAL": { - 'access': '', - 'area': 'yes', - 'bicycle': '', - 'bridge': '', - 'foot': ['ft_yes', 'foot_designated'] -}} +from wahoomc.constants_functions import translate_tags_to_keep class TestTranslateTags(unittest.TestCase): @@ -30,54 +13,8 @@ class TestTranslateTags(unittest.TestCase): tests for translating tags-constants between the universal format and OS-specific formats """ - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_simple_macos(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to macOS - """ - tags = ['access', 'area=yes'] - mock_json_load.return_value = tags_universal_simple - - transl_tags = translate_tags_to_keep() - self.assertEqual(tags, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_simple_win(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to Windows - """ - tags_win = 'access= area=yes' - mock_json_load.return_value = tags_universal_simple - - transl_tags = translate_tags_to_keep(sys_platform='Windows') - self.assertEqual(tags_win, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_adv_macos(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to macOS - """ - tags = ['access', 'area=yes', 'bicycle', - 'bridge', 'foot=ft_yes, foot_designated'] - mock_json_load.return_value = tags_universal_adv - - transl_tags = translate_tags_to_keep() - self.assertEqual(tags, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_adv_win(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to Windows - """ - tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated' - mock_json_load.return_value = tags_universal_adv - - transl_tags = translate_tags_to_keep(sys_platform='Windows') - self.assertEqual(tags_win, transl_tags) + def setUp(self): + self.tags_to_keep = os.path.join(os.path.dirname(__file__), 'resources', 'tags-to-keep.json') def test_translate_tags_to_keep_full_macos(self): """ @@ -91,7 +28,7 @@ def test_translate_tags_to_keep_full_macos(self): 'leisure=park, nature_reserve', 'railway=rail, tram, station, stop', 'surface', 'tracktype', 'tunnel', 'waterway=canal, drain, river, riverbank, stream', 'wood=deciduous', 'tourism=alpine_hut'] - transl_tags = translate_tags_to_keep(use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep) self.assertEqual(tags, transl_tags) def test_translate_tags_to_keep_full_win(self): @@ -100,8 +37,7 @@ def test_translate_tags_to_keep_full_win(self): """ tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated amenity=fuel =cafe =drinking_water =shelter shop=bakery =bicycle highway=abandoned =bus_guideway =disused =bridleway =byway =construction =cycleway =footway =living_street =motorway =motorway_link =path =pedestrian =primary =primary_link =residential =road =secondary =secondary_link =service =steps =tertiary =tertiary_link =track =trunk =trunk_link =unclassified natural=coastline =nosea =sea =beach =land =scrub =water =wetland =wood landuse=forest =commercial =industrial =residential =retail leisure=park =nature_reserve railway=rail =tram =station =stop surface= tracktype= tunnel= waterway=canal =drain =river =riverbank =stream wood=deciduous tourism=alpine_hut' - transl_tags = translate_tags_to_keep( - sys_platform='Windows', use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, osmium=False) self.assertEqual(tags_win, transl_tags) def test_translate_name_tags_to_keep_full_macos(self): @@ -111,7 +47,7 @@ def test_translate_name_tags_to_keep_full_macos(self): names_tags = ['admin_level=2', 'area=yes', 'mountain_pass', 'natural', 'place=city, hamlet, island, isolated_dwelling, islet, locality, suburb, town, village, country'] - transl_tags = translate_tags_to_keep(name_tags=True, use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, name_tags=True) self.assertEqual(names_tags, transl_tags) def test_translate_name_tags_to_keep_full_win(self): @@ -121,33 +57,9 @@ def test_translate_name_tags_to_keep_full_win(self): names_tags_win = 'admin_level=2 area=yes mountain_pass= natural= place=city =hamlet =island =isolated_dwelling =islet =locality =suburb =town =village =country' - transl_tags = translate_tags_to_keep( - name_tags=True, sys_platform='Windows', use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) self.assertEqual(names_tags_win, transl_tags) -class TestTagWahooXML(unittest.TestCase): - """ - tests for tag-wahoo xml file - """ - - def test_not_existing_tag_wahoo_xml(self): - """ - check if a non-existing tag-wahoo xml file issues an exception - """ - with self.assertRaises(TagWahooXmlNotFoundError): - get_tag_wahoo_xml_path("not_existing.xml") - - def test_existing_tag_wahoo_xml(self): - """ - check if the correct path of an existing tag-wahoo xml file is returned - """ - - expected_path = os.path.join( - RESOURCES_DIR, "tag_wahoo_adjusted", "tag-wahoo-poi.xml") - self.assertEqual(get_tag_wahoo_xml_path( - "tag-wahoo-poi.xml"), expected_path) - - if __name__ == '__main__': unittest.main() diff --git a/tests/test_downloader.py b/tests/test_downloader.py index c8b3ba55..475ef49e 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -186,7 +186,7 @@ def test_download_macos_files(self): """ if platform.system() != "Windows": path = os.path.join(str(constants.USER_DIR), '.openstreetmap', 'osmosis', - 'plugins', 'mapsforge-map-writer-0.21.0-jar-with-dependencies.jar') + 'plugins', 'mapsforge-map-writer-0.25.0-jar-with-dependencies.jar') if os.path.exists(path): os.remove(path) diff --git a/tests/test_generated_files.py b/tests/test_generated_files.py index 92c8b9e4..aac7dffb 100644 --- a/tests/test_generated_files.py +++ b/tests/test_generated_files.py @@ -7,6 +7,7 @@ from os import walk import platform import shutil +import sys import unittest import subprocess @@ -214,11 +215,11 @@ def run_wahoomapscreator_cli(self, country, hdd_mode=False): if not hdd_mode: # run processing of input-country via CLI in standard mode result = os.system( - f'python -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc') + f'{sys.executable} -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc') else: # run processing of input-country via CLI in mapwriter hdd mode result = os.system( - f'python -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc -hd') + f'{sys.executable} -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc -hdd') # check if run was successful self.assertEqual(result, 0) diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index 9eb8db04..962fa005 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -14,7 +14,16 @@ from wahoomc import constants -class TestOsmMapsCalculation(unittest.TestCase): +class TestOsmMaps(unittest.TestCase): + """ + base test class + """ + + def setUp(self): + self.tags_to_keep = os.path.join(constants.RESOURCES_DIR, 'tags-to-keep.json') + self.tag_transform = os.path.join(constants.RESOURCES_DIR, 'tunnel-transform.xml') + +class TestOsmMapsCalculation(TestOsmMaps): """ tests for the OSM maps file """ @@ -148,7 +157,8 @@ def process_and_check_border_countries(self, inp_val, calc_border_c, exp_result, if inp_mode == 'country': o_input_data.country = inp_val o_osm_data = CountryOsmData(o_input_data) - elif inp_mode == 'xy_coordinate': + else: + self.assertEqual(inp_mode, 'xy_coordinate') o_input_data.xy_coordinates = inp_val o_osm_data = XYOsmData(o_input_data) @@ -163,7 +173,7 @@ def process_and_check_border_countries(self, inp_val, calc_border_c, exp_result, self.assertEqual(result, exp_result) -class TestOSMMapsInput(unittest.TestCase): +class TestOSMMapsInput(TestOsmMaps): """ tests for input of OsmData """ @@ -184,7 +194,7 @@ def test_folder_name_many_countries(self): """ o_osm_data = self.get_osm_data_instance('albania,alps,andorra,austria,azores,belarus,belgium,bosnia-herzegovina,britain-and-ireland,bulgaria,croatia,cyprus,czech-republic,dach,denmark,estonia,faroe-islands,finland,france,georgia,germany,great-britain,greece,guernsey-jersey,hungary,iceland,ireland-and-northern-ireland,isle-of-man,italy,kosovo,latvia,liechtenstein,lithuania,luxembourg,macedonia,malta,moldova,monaco,montenegro,netherlands,norway,poland,portugal,romania,serbia,slovakia,slovenia,spain,sweden,switzerland,turkey,ukraine') - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep, self.tag_transform) folder_name = o_osm_maps.calculate_folder_name('.map.lzma') folder_name_maps = o_osm_maps.calculate_folder_name('.map') @@ -215,7 +225,7 @@ def get_osm_data_instance(self, country_input): return o_osm_data -class TestConfigFile(unittest.TestCase): +class TestConfigFile(TestOsmMaps): """ tests for the config .json file in the "wahooMapsCreatorData/_tiles/{country}" directory """ @@ -238,7 +248,7 @@ def test_version_and_tags_of_country_config_file(self): # download files marked for download to fill up map_file per country to write to config o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep, self.tag_transform) o_osm_maps.write_country_config_file(o_input_data.country) diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 012b8e3e..53e2d59d 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -18,39 +18,18 @@ log = logging.getLogger('main-logger') -class TagWahooXmlNotFoundError(Exception): - """Raised when the specified tag-wahoo xml file does not exist""" - - class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" -def translate_tags_to_keep(name_tags=False, sys_platform='', use_repo=False): +def translate_tags_to_keep(tags_to_keep, name_tags=False, osmium=True): """ translates the given tags to format of the operating system. """ - - if sys_platform == "Windows": - separator = ' =' - else: - separator = ', ' - tags_modif = [] - # read tags-to-keep .json from user-dir in favor of python installation - # evaluate path first: user-dir in favor of PyPI installation - if not use_repo: - for path in get_absolute_dir_user_or_repo('', file='tags-to-keep.json'): - if os.path.exists(path): - break - # force using file from repo - used in unittests for equal output - else: - path = get_absolute_dir_user_or_repo( - '', file='tags-to-keep.json')[1] - - # read the tags from the evaluated path above - tags_from_json = read_json_file_generic(path) + # read the tags from the passed tags_to_keep path + tags_from_json = read_json_file_generic(tags_to_keep) if not tags_from_json: raise TagsToKeepNotFoundError @@ -61,21 +40,22 @@ def translate_tags_to_keep(name_tags=False, sys_platform='', use_repo=False): universal_tags = tags_from_json['NAME_TAGS_TO_KEEP_UNIVERSAL'] for tag, value in universal_tags.items(): - to_append = transl_tag_value(sys_platform, separator, tag, value) + to_append = transl_tag_value(osmium, tag, value) tags_modif.append(to_append) - if sys_platform == "Windows": + if not osmium: tags_modif = ' '.join(tags_modif) return tags_modif -def transl_tag_value(sys_platform, separator, tag, value): +def transl_tag_value(osmium, tag, value): """ translates one tag with value(s) to a "common" format """ if isinstance(value, list): + separator = ', ' if osmium else ' =' for iteration, sing_val in enumerate(value): if iteration == 0: to_append = f'{tag}={sing_val}' @@ -84,10 +64,7 @@ def transl_tag_value(sys_platform, separator, tag, value): elif value: to_append = f'{tag}={value}' else: - if sys_platform == "Windows": - to_append = f'{tag}=' - else: - to_append = tag + to_append = tag if osmium else f'{tag}=' return to_append @@ -112,21 +89,6 @@ def get_tooling_win_path(path_in_tooling_win, in_user_dir=False): # all other "toolings": concatenate with win tooling dir return os.path.join(tooling_dir, path_in_tooling_win) - -def get_tag_wahoo_xml_path(tag_wahoo_xml): - """ - return path to tag-wahoo xml file if the file exists - - from the user directory "USER_WAHOO_MC/_config/tag_wahoo_adjusted/tag_wahoo_xml" - - 2ndly from the PyPI installation: "RESOURCES_DIR/tag_wahoo_adjusted/tag_wahoo_xml" - """ - - for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", tag_wahoo_xml): - if os.path.exists(path): - return path - - raise TagWahooXmlNotFoundError - - def get_absolute_dir_user_or_repo(folder, file=''): """ return the absolute path to the folder (and file) in this priorization diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index bed2d948..57372b9c 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -119,8 +119,8 @@ def download_tooling(): check here for new mapwriter plugin version: https://github.com/mapsforge/mapsforge """ - map_writer_filename = 'mapsforge-map-writer-0.21.0-jar-with-dependencies.jar' - mapwriter_plugin_url = 'https://search.maven.org/remotecontent?filepath=org/mapsforge/mapsforge-map-writer/0.21.0/' + map_writer_filename + map_writer_filename = 'mapsforge-map-writer-0.25.0-jar-with-dependencies.jar' + mapwriter_plugin_url = 'https://search.maven.org/remotecontent?filepath=org/mapsforge/mapsforge-map-writer/0.25.0/' + map_writer_filename # Windows if platform.system() == "Windows": @@ -180,7 +180,7 @@ def get_latest_pypi_version(): response = requests.get( 'https://pypi.org/pypi/wahoomc/json', timeout=1) return response.json()['info']['version'] - except (requests.ConnectionError, requests.Timeout): + except (requests.ConnectionError, requests.Timeout, requests.exceptions.JSONDecodeError): return None diff --git a/wahoomc/input.py b/wahoomc/input.py index 25bc55bc..450be9a3 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -5,6 +5,7 @@ # import official python packages import argparse +import os import sys from platform import uname @@ -13,6 +14,7 @@ from tkinter import ttk # import custom python packages +from wahoomc.constants_functions import get_absolute_dir_user_or_repo from wahoomc.geofabrik_json import GeofabrikJson from wahoomc.geofabrik_json import CountyIsNoGeofabrikCountry from wahoomc.geofabrik import CountryGeofabrik @@ -78,7 +80,13 @@ def process_call_of_the_tool(): # Save uncompressed maps for Cruiser if True options_args.add_argument('-c', '--cruiser', action='store_true', help="save uncompressed maps for Cruiser") - # specify the file with tags to keep in the output // file needs to be in wahoo_mc/resources/tag_wahoo_adjusted + # specify the file with tags to keep when filtering + options_args.add_argument('--tags_to_keep', default=InputData().tags_to_keep, + help="file with tags to keep when filtering") + # specify the file with tag-transform rules + options_args.add_argument('--tag_transform', default=InputData().tag_transform, + help="file with tag-transform rules") + # specify the file with tags to keep in the output options_args.add_argument('-tag', '--tag_wahoo_xml', default=InputData().tag_wahoo_xml, help="file with tags to keep in the output") # zip the country (and country-maps) folder @@ -114,6 +122,8 @@ def process_call_of_the_tool(): o_input_data.force_download = args.forcedownload o_input_data.force_processing = args.forceprocessing + o_input_data.tags_to_keep = args.tags_to_keep + o_input_data.tag_transform = args.tag_transform o_input_data.tag_wahoo_xml = args.tag_wahoo_xml o_input_data.save_cruiser = args.cruiser o_input_data.zip_folder = args.zip @@ -194,6 +204,8 @@ def __init__(self): self.contour = False self.use_srtm1 = False + self.tags_to_keep = "tags-to-keep.json" + self.tag_transform = "tunnel-transform.xml" self.tag_wahoo_xml = "tag-wahoo-poi.xml" self.zip_folder = False self.save_cruiser = False @@ -201,7 +213,8 @@ def __init__(self): self.verbose = False - def is_required_input_given_or_exit(self, issue_message): + + def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches """ check, if the minimal required arguments is given: - country @@ -210,25 +223,47 @@ def is_required_input_given_or_exit(self, issue_message): If not, depending on the import parameter, the """ if (self.country in ('None', '') and self.xy_coordinates in ('None', '')): - if issue_message: - sys.exit("Nothing to do. Start with -h or --help to see command line options." - "Or in the GUI select a country to create maps for.") - else: - sys.exit() - elif self.country and self.xy_coordinates: - sys.exit( - "Country and X/Y coordinates are given. Only one of both is allowed!") - elif self.country: + sys.exit("Nothing to do. Start with -h or --help to see command line options." + "Or in the GUI select a country to create maps for.") + + if self.country and self.xy_coordinates: + sys.exit("Country and X/Y coordinates are given. Only one of both is allowed!") + + if self.country: # countries = try: CountryGeofabrik.split_input_to_list(self.country) except CountyIsNoGeofabrikCountry as exception: sys.exit(exception) - # if we made it until here, sys.exit() was not called and therefore all countries OK ;-) - return True - else: - return True + if not os.path.exists(self.tags_to_keep): + for path in get_absolute_dir_user_or_repo("", self.tags_to_keep): + if os.path.exists(path): + self.tags_to_keep = path + break + + if not os.path.exists(self.tags_to_keep): + sys.exit(f'The tags-to-keep json file was not found: \"{self.tags_to_keep}\"') + + if not os.path.exists(self.tag_transform): + for path in get_absolute_dir_user_or_repo("", self.tag_transform): + if os.path.exists(path): + self.tag_transform = path + break + + if not os.path.exists(self.tag_transform): + sys.exit(f'The tag-transform xml file was not found: \"{self.tag_transform}\"') + + if not os.path.exists(self.tag_wahoo_xml): + for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", self.tag_wahoo_xml): + if os.path.exists(path): + self.tag_wahoo_xml = path + break + + if not os.path.exists(self.tag_wahoo_xml): + sys.exit(f'The tag-wahoo xml file was not found: \"{self.tag_wahoo_xml}\"') + + return True class GuiInput(tk.Tk): @@ -250,7 +285,7 @@ def start_gui(self): # start GUI self.mainloop() - self.o_input_data.is_required_input_given_or_exit(issue_message=True) + self.o_input_data.is_required_input_given_or_exit() return self.o_input_data def build_gui(self): diff --git a/wahoomc/main.py b/wahoomc/main.py index dfa17b11..5fd250ef 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -18,6 +18,8 @@ from wahoomc.osm_maps_functions import OsmMaps from wahoomc.osm_data import CountryOsmData, XYOsmData +log = logging.getLogger('main-logger') + # logging used in the terminal output: # # means top-level command # ! means error @@ -47,21 +49,26 @@ def run(run_level): o_input_data = process_call_of_the_tool() if o_input_data.verbose: - logging.getLogger().setLevel(logging.DEBUG) + log.setLevel(logging.DEBUG) if run_level == 'init': copy_jsons_from_repo_to_user('tag_wahoo_adjusted') copy_jsons_from_repo_to_user('.', 'tags-to-keep.json') else: # Is there something to do? - o_input_data.is_required_input_given_or_exit(issue_message=True) + o_input_data.is_required_input_given_or_exit() if o_input_data.contour: check_installation_of_programs_credentials_for_contour_lines() + log.info('# Used configuration') + log.info('+ tags-to-keep file: %s', o_input_data.tags_to_keep) + log.info('+ tag-transform file: %s', o_input_data.tag_transform) + log.info('+ map-writer tag-conf file: %s', o_input_data.tag_wahoo_xml) + if o_input_data.country: o_osm_data = CountryOsmData(o_input_data) - elif o_input_data.xy_coordinates: + else: o_osm_data = XYOsmData(o_input_data) timings = Timings() @@ -72,7 +79,7 @@ def run(run_level): # Download files marked for download o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, o_input_data.tags_to_keep, o_input_data.tag_transform) # Filter tags from country osm.pbf files' o_osm_maps.filter_tags_from_country_osm_pbf_files() diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 94128805..91fb56e8 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -16,8 +16,7 @@ # import custom python packages from wahoomc.file_directory_functions import read_json_file_country_config, create_empty_directories, write_json_file_generic -from wahoomc.constants_functions import translate_tags_to_keep, \ - get_tooling_win_path, get_tag_wahoo_xml_path, TagWahooXmlNotFoundError +from wahoomc.constants_functions import translate_tags_to_keep, get_tooling_win_path from wahoomc.setup_functions import read_earthexplorer_credentials @@ -38,14 +37,13 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""): """ run given cmd-subprocess and issue error message if wished """ + log.debug('running subprocess: %s', str(cmd)) if not cwd: process = subprocess.run( cmd, capture_output=True, text=True, encoding="utf-8", check=False) else: - process = subprocess.run( # pylint: disable=consider-using-with - cmd, capture_output=True, cwd=cwd, text=True, encoding="utf-8", check=False) - + process = subprocess.run(cmd, capture_output=True, cwd=cwd, text=True, encoding="utf-8", check=False) if error_message and process.returncode != 0: # 0 means success log.error('subprocess error output:') @@ -77,8 +75,10 @@ class OsmMaps: # Number of workers for the Osmosis read binary fast function workers = '1' - def __init__(self, o_osm_data): + def __init__(self, o_osm_data, tags_to_keep, tag_transform): self.o_osm_data = o_osm_data + self.tags_to_keep = tags_to_keep + self.tag_transform = tag_transform self.osmconvert_path = get_tooling_win_path('osmconvert') create_empty_directories( @@ -133,10 +133,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st '+ Filtering unwanted map objects out of map of %s', key) cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append( - '--keep="' + translate_tags_to_keep(sys_platform=platform.system()) + '"') - cmd.append('--keep-tags="all type= layer= ' + - translate_tags_to_keep(sys_platform=platform.system()) + '"') + cmd.append('--keep="' + translate_tags_to_keep(self.tags_to_keep, osmium=False) + '"') + cmd.append('--keep-tags="all type= layer= ' + translate_tags_to_keep(self.tags_to_keep, osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_win) run_subprocess_and_log_output( @@ -144,12 +142,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append( - '--keep="' + translate_tags_to_keep( - name_tags=True, sys_platform=platform.system()) + '"') - cmd.append('--keep-tags="all type= name= layer= ' + - translate_tags_to_keep( - name_tags=True, sys_platform=platform.system()) + '"') + cmd.append('--keep="' + translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) + '"') + cmd.append('--keep-tags="all type= name= layer= ' + translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_names_win) run_subprocess_and_log_output( @@ -176,8 +170,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep( - sys_platform=platform.system())) + cmd.extend(translate_tags_to_keep(self.tags_to_keep)) cmd.extend(['-o', out_file_pbf_filtered_mac]) cmd.append('--overwrite') @@ -186,8 +179,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep( - name_tags=True, sys_platform=platform.system())) + cmd.extend(translate_tags_to_keep(self.tags_to_keep, name_tags=True)) cmd.extend(['-o', out_file_pbf_filtered_names_mac]) cmd.append('--overwrite') @@ -241,16 +233,8 @@ def generate_land(self): # create land1.osm if not os.path.isfile(out_file_land1+'1.osm') or self.o_osm_data.force_processing is True: - # Windows - if platform.system() == "Windows": - cmd = ['python', os.path.join(RESOURCES_DIR, - 'shape2osm.py'), '-l', out_file_land1, land_file] - - # Non-Windows - else: - cmd = ['python', os.path.join(RESOURCES_DIR, - 'shape2osm.py'), '-l', out_file_land1, land_file] - + cmd = [sys.executable, os.path.join(RESOURCES_DIR, + 'shape2osm.py'), '-l', out_file_land1, land_file] run_subprocess_and_log_output( cmd, f'! Error creating land.osm for tile: {tile["x"]},{tile["y"]}') self.log_tile_debug(tile["x"], tile["y"], tile_count, timings_tile.stop_and_return()) @@ -450,7 +434,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto log.info('# Merge splitted tiles with land, elevation, and sea') timings = Timings() tile_count = 1 - for tile in self.o_osm_data.tiles: # pylint: disable=too-many-nested-blocks + for tile in self.o_osm_data.tiles: self.log_tile_info(tile["x"], tile["y"], tile_count) timings_tile = Timings() @@ -505,8 +489,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto cmd.extend( ['--rx', 'file='+os.path.join(out_tile_dir, 'sea.osm'), '--s', '--m']) - cmd.extend(['--tag-transform', 'file=' + os.path.join(RESOURCES_DIR, - 'tunnel-transform.xml'), '--wb', out_file_merged, 'omitmetadata=true']) + cmd.extend(['--tag-transform', 'file=' + self.tag_transform, '--wb', out_file_merged, 'omitmetadata=true']) run_subprocess_and_log_output( cmd, f'! Error in Osmosis with tile: {tile["x"]},{tile["y"]}') @@ -545,7 +528,7 @@ def sort_osm_files(self, tile): log.debug('+ Sorting land* osm files: OK') - def create_map_files(self, save_cruiser, tag_wahoo_xml, hdd_mode): + def create_map_files(self, save_cruiser, tag_conf_file, hdd_mode): """ Creating .map files """ @@ -584,16 +567,10 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml, hdd_mode): f'bbox={tile["bottom"]:.6f},{tile["left"]:.6f},{tile["top"]:.6f},{tile["right"]:.6f}') cmd.append('zoom-interval-conf=12,0,17') cmd.append(f'threads={threads}') + cmd.append('tag-values=true') if hdd_mode: cmd.append('type=hd') - # add path to tag-wahoo xml file - try: - cmd.append( - f'tag-conf-file={get_tag_wahoo_xml_path(tag_wahoo_xml)}') - except TagWahooXmlNotFoundError: - log.error( - 'The tag-wahoo xml file was not found: ˚%s˚. Does the file exist and is your input correct?', tag_wahoo_xml) - sys.exit() + cmd.append(f'tag-conf-file={tag_conf_file}') run_subprocess_and_log_output( cmd, f'Error in creating map file via Osmosis with tile: {tile["x"]},{tile["y"]}. mapwriter plugin installed?') @@ -733,8 +710,8 @@ def write_country_config_file(self, country): configuration = { "version_last_run": VERSION, "changed_ts_map_last_run": get_timestamp_last_changed(self.o_osm_data.border_countries[country]['map_file']), - "tags_last_run": translate_tags_to_keep(sys_platform=platform.system()), - "name_tags_last_run": translate_tags_to_keep(name_tags=True, sys_platform=platform.system()) + "tags_last_run": translate_tags_to_keep(self.tags_to_keep), + "name_tags_last_run": translate_tags_to_keep(self.tags_to_keep, name_tags=True) } write_json_file_generic(os.path.join( @@ -749,8 +726,8 @@ def tags_are_identical_to_last_run(self, country): try: country_config = read_json_file_country_config(os.path.join( USER_OUTPUT_DIR, country, ".config.json")) - if not country_config["tags_last_run"] == translate_tags_to_keep(sys_platform=platform.system()) \ - or not country_config["name_tags_last_run"] == translate_tags_to_keep(name_tags=True, sys_platform=platform.system()): + if not country_config["tags_last_run"] == translate_tags_to_keep(self.tags_to_keep) \ + or not country_config["name_tags_last_run"] == translate_tags_to_keep(self.tags_to_keep, name_tags=True): tags_are_identical = False except (FileNotFoundError, KeyError): tags_are_identical = False @@ -785,7 +762,7 @@ def log_tile_debug(self, tile_x, tile_y, tile_count, additional_info=''): """ self.log_tile(tile_x, tile_y, tile_count, True, additional_info) - def log_tile(self, tile_x, tile_y, tile_count, log_level_debug, additional_info=''): # pylint: disable=too-many-arguments + def log_tile(self, tile_x, tile_y, tile_count, log_level_debug, additional_info=''): # pylint: disable=too-many-arguments,too-many-positional-arguments """ unified status logging for this class """ diff --git a/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml b/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml index 6f43434f..8fe06d45 100644 --- a/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml +++ b/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml @@ -171,11 +171,6 @@ - - - - - diff --git a/wahoomc/setup_functions.py b/wahoomc/setup_functions.py index ad735ed7..7d38e66b 100644 --- a/wahoomc/setup_functions.py +++ b/wahoomc/setup_functions.py @@ -45,7 +45,7 @@ def adjustments_due_to_breaking_changes(): """ handle breaking changes """ - version_last_run = read_version_last_run() # pylint: disable=unused-variable + version_last_run = read_version_last_run() # Osmosis in v.0.49.2 seams not to be working on WINDOWS since the upgrade to v0.49.2 # - due to the path into it was downloaded, 'tooling_win/Osmosis/osmosis-0.49.2'