From e69174d261d299fa7bed05516981418b5b8bc08e Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Fri, 21 Aug 2026 13:58:33 +0200 Subject: [PATCH 01/13] Simplify `fiboa publish` for catalog-driven publication `fiboa publish` now only converts, validates, builds PMTiles and writes a collection.json with relative links, file:size/file:checksum (multihash), a web-map-links v1.3.0 `pmtiles` link with `pmtiles:layers` and a `visual` asset. README/LICENSE generation, the data-survey lookup and the S3 upload are gone; catalogs such as fieldsoftheworld/harmonized-field-data-catalog own those. spdx-license-list is only needed by tests and moves to the dev feature. Also: - FiboaBaseConverter: keep the determination:datetime column that `use_variant_as_determination` adds; it was removed again as unlisted (affected dk, hr). - be_vlg: drop plots without a crop code (one in 2023 failed validation), take the determination date from the variant year, add the 2026 edition. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 3 + README.md | 47 +-- fiboa_cli/conversion/fiboa_converter.py | 4 + fiboa_cli/datasets/be_vlg.py | 10 +- fiboa_cli/publish.py | 490 +++++++--------------- pixi.lock | 11 +- pyproject.toml | 2 +- tests/data-files/publish/BE-VLG-survey.md | 78 ---- tests/test_publish.py | 49 ++- 9 files changed, 215 insertions(+), 479 deletions(-) delete mode 100644 tests/data-files/publish/BE-VLG-survey.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b3747e4..7a80ad1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +- Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) +- BE-VLG: drop plots without a crop code (one such plot in the 2023 edition made validation fail); derive determination:datetime from the variant year instead of a constant date +- `fiboa publish` no longer uploads to S3 or generates README/LICENSE files. It creates GeoParquet, PMTiles and a STAC Collection with relative links, `file:size`/`file:checksum` and a web-map-links v1.3.0 `pmtiles` link. Publishing is done by catalogs such as the [harmonized field data catalog](https://github.com/fieldsoftheworld/harmonized-field-data-catalog). - Add Italy Tuscany (IT-1) basd on EuroCrops v2 - Suuport multiple years for CZ - Multiple years for DE_sh diff --git a/README.md b/README.md index 57982654..9a5f5252 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ fiboa CLI supports various commands to work with the files: - [Improve a fiboa Parquet file](#improve-a-fiboa-parquet-file) - [Update an extension template with new names](#update-an-extension-template-with-new-names) - [Converter for existing datasets](#converter-for-existing-datasets) - - [Publish datasets to source coop or your own s3 repository](#publish-datasets-to-source-coop-or-your-own-s3-repository) + - [Publish datasets](#publish-datasets) - [Development](#development) - [Implement a converter](#implement-a-converter) - [Run in Docker](#run-in-docker) @@ -193,46 +193,43 @@ Use any of the IDs from the list to convert an existing dataset to fiboa: See [Implement a converter](#implement-a-converter) for details about how to -### Publish datasets to source coop or your own s3 repository +### Publish datasets `fiboa publish -o ` -The publish converts and publishes a fiboa dataset to source coop or your own s3 repository. The target directory -will be filled with the following files: +Converts and validates a fiboa dataset and prepares everything that is needed to publish it +in a (STAC-based) catalog. The target directory will be filled with the following files: ``` / - .parquet - .pmtiles # requires working ogr2ogr and tippecanoe - stac/collection.json - README.md # generated if --generate-meta/-gm flag is present - LICENSE.txt # generated if --generate-meta/-gm flag is present + [-].parquet + [-].pmtiles # requires working ogr2ogr and tippecanoe + collection.json # STAC Collection with relative links to the files above ``` -This directory is synchronized to the s3 repository (default source.coop/fiboa/data). +The STAC Collection carries `file:size` and `file:checksum` for the files and a +`pmtiles` link (web-map-links extension). Existing files in the target directory are reused, +delete them to regenerate. Uploading to a bucket and catalog-specific metadata +(README, styles, thumbnails, ...) are the job of the catalog that publishes the data, e.g. the +[harmonized field data catalog](https://github.com/fieldsoftheworld/harmonized-field-data-catalog). -**Requirements**: Requires the [aws CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) to be installed, -and `AWS_ACCESS_KEY_ID` with `AWS_SECRET_ACCESS_KEY` environment variables. Also, for generating the pmtiles file, -it requires [ogr2ogr](https://gdal.org/programs/ogr2ogr.html) and [tippecanoe](https://github.com/mapbox/tippecanoe). +The command runs: -The command executes the following steps: - -- `fiboa convert` to generate a fiboa parquet dataset. All convert parameters are passed to the converter. -- `fiboa validate` to validate the fiboa dataset -- creates a .pmtiles from the parquet file. Uses ogr2ogr and tippecanoe -- `fiboa create-stac-collection` to create a STAC collection -- `fiboa publish` to publish the fiboa dataset to a source coop or your own s3 repository +- `fiboa convert` to create a `[-].parquet` file +- `fiboa validate` to validate the GeoParquet file +- `ogr2ogr | tippecanoe` to create the PMTiles file +- `fiboa create-stac-collection` to create the STAC Collection Examples: -- `fiboa publish at_crop -o data/at_crop` -- `fiboa publish -c /tmp/cache -gm br_conab -o data/br_conab` +- `fiboa publish at -o data/at` +- `fiboa publish -c /tmp/cache nl --variant 2025 -o data/nl/2025` Relevant parameters: -- `--generate-meta/-gm` Generatse the README.md and LICENSE.txt files if absent, based on data-survey and converter properties. -- `--data-url` The URL to the data repository, used when generating the README -- `--s3-upload-path` The `aws s3 sync` target. Defaults to `s3://source.coop/fiboa/data` . Uploading requires the `aws` CLI, and `AWS_ACCESS_KEY_ID` with `AWS_SECRET_ACCESS_KEY` environment variables. +- `--variant` Choose the variant (e.g. year) of a dataset, defaults to the first variant. +- `--no-pmtiles` Skip PMTiles generation. +- `--tippecanoe-opts` Options passed to tippecanoe, defaults to `-zg --drop-densest-as-needed --extend-zooms-if-still-dropping`. Check `fiboa publish --help` for more details. diff --git a/fiboa_cli/conversion/fiboa_converter.py b/fiboa_cli/conversion/fiboa_converter.py index c5ac7227..a073b0e5 100644 --- a/fiboa_cli/conversion/fiboa_converter.py +++ b/fiboa_cli/conversion/fiboa_converter.py @@ -14,6 +14,10 @@ class FiboaBaseConverter(BaseConverter): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.extensions.add(get_fiboa_uri()) + if self.use_variant_as_determination: + # The column is added in post_migrate; list it so it survives the + # "remove unlisted columns" step of the base converter. + self.columns = {**self.columns, "determination:datetime": "determination:datetime"} def post_migrate(self, gdf): gdf = super().post_migrate(gdf) diff --git a/fiboa_cli/datasets/be_vlg.py b/fiboa_cli/datasets/be_vlg.py index 4d4c253a..b8dc6d9a 100644 --- a/fiboa_cli/datasets/be_vlg.py +++ b/fiboa_cli/datasets/be_vlg.py @@ -10,6 +10,7 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): variants = { str(k): {PREFIX + v: [v.replace("_GPKG.zip", ".gpkg")]} for k, v in ( + (2026, "agpa_2026_2026-06-02_public.zip"), (2025, "Landbouwgebruikspercelen_2025_-_Voorlopig_(extractie_02-06-2025)_GPKG.zip"), (2024, "Landbouwgebruikspercelen_2024_-_Definitief_(extractie_27-03-2025)_GPKG.zip"), (2023, "Landbouwgebruikspercelen_2023_-_Definitief_(extractie_28-03-2024)_GPKG.zip"), @@ -42,8 +43,13 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): "GWSCOD_H": "crop:code", "GWSNAM_H": "crop:name", } - column_additions = { - "determination:datetime": "2024-03-28T00:00:00Z", + # Each edition is the campaign year of its variant; the old constant + # "2024-03-28" was the extraction date of one edition applied to all of them. + use_variant_as_determination = True + column_filters = { + # A handful of plots (e.g. one in 2023, typology "Niet-geclassificeerd") carry no + # crop code; crop:code is required by the crop extension, so drop them. + "GWSCOD_H": lambda col: col.notna(), } ec_mapping_csv = "be_vlg_2021.csv" diff --git a/fiboa_cli/publish.py b/fiboa_cli/publish.py index 122cc4da..20eddeab 100644 --- a/fiboa_cli/publish.py +++ b/fiboa_cli/publish.py @@ -1,17 +1,14 @@ +import hashlib import json import os -import re +import shutil +import subprocess import sys -from datetime import date -from functools import cache from pathlib import Path import click -import requests -import spdx_license_list from vecorel_cli.basecommand import BaseCommand, runnable from vecorel_cli.cli.options import VECOREL_TARGET -from vecorel_cli.encoding.auto import create_encoding from .convert import ConvertData from .converters import Converters @@ -19,84 +16,50 @@ from .registry import Registry from .validate import ValidateData -STAC_EXTENSION = "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" -DESCRIPTIONS = { - "id": "Unique identifier", - "collection": "The collection identifier", - "inspire:id": "The INSPIRE identifier", - "determination:datetime": "Timestamp of the determination of the field boundary", - "metrics:area": "Field area in square meters", - "metrics:perimeter": "Field perimeter in square meters", - "crop:code_list": "A link to the code list", - "crop:code": "The crop code", - "crop:name": "Crop name in the original language", - "crop:name_en": "Crop name in English", - "hcat:name": "The machine-readable HCAT name of the crop", - "hcat:code": "The 10-digit HCAT code indicating the hierarchy of the crop", - "hcat:name_en": "The HCAT crop name translated into English", - "admin:country_code": "ISO 3166-1 alpha-2 country code.", - "admin:subdivision_code": "ISO 3166-2 principal subdivision code (e.g. province or state)", -} +FILE_EXTENSION = "https://stac-extensions.github.io/file/v2.1.0/schema.json" +WEB_MAP_LINKS_EXTENSION = "https://stac-extensions.github.io/web-map-links/v1.3.0/schema.json" +PMTILES_MEDIA_TYPE = "application/vnd.pmtiles" +TIPPECANOE_DEFAULT_OPTS = "-zg --drop-densest-as-needed --extend-zooms-if-still-dropping" is_windows = os.name == "nt" +def multihash_sha256(path: Path, chunk_size: int = 1024 * 1024) -> str: + """ + sha2-256 multihash of a file, hex encoded: 0x12 (sha2-256), 0x20 (32 bytes), digest. + This is the encoding the STAC file extension expects for ``file:checksum``. + """ + digest = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + digest.update(chunk) + return "1220" + digest.hexdigest() + + class Publish(BaseCommand): cmd_name = "publish" - cmd_help = f"Convert and publish a {Registry.project} dataset to source coop." - url_base = "https://data.source.coop/fiboa/data" + cmd_help = ( + f"Convert a {Registry.project} dataset and prepare it for publication: " + "GeoParquet, PMTiles and a STAC Collection with relative links." + ) @staticmethod def get_cli_args(): return { **ConvertData.get_cli_args(), "target": VECOREL_TARGET(folder=True), - "generate_meta": click.option( - "--generate-meta", - "-gm", - is_flag=True, - type=click.BOOL, - help="Generate README.txt and LICENSE.txt for the dataset if not present.", - default=False, - ), - "data_url": click.option( - "--data-url", - type=click.STRING, - help="When generating documentation, this is the link to the data.", - ), - "s3_upload_path": click.option( - "--s3-upload-path", - type=click.STRING, - help="Upload to this path on S3. By default it's the source coop fiboa data repository.", - ), - "yes": click.option( - "--yes", - "-y", + "pmtiles": click.option( + "--pmtiles/--no-pmtiles", is_flag=True, - type=click.BOOL, - help="Answer yes to all questions.", - default=False, - show_default=True, - ), - "data_survey_url": click.option( - "--data-survey-url", - type=click.STRING, - help="URL to the data survey markdown file.", - default=os.getenv("FIBOA_DATA_SURVEY"), + help="Generate PMTiles with ogr2ogr and tippecanoe.", + default=True, show_default=True, ), - "editor": click.option( - "--editor", + "tippecanoe_opts": click.option( + "--tippecanoe-opts", type=click.STRING, - help="Editor to use when editing generated files.", - default=os.getenv("EDITOR", "edit" if is_windows else "nano"), - show_default=True, - ), - "converted_by": click.option( - "--converted-by", - type=click.STRING, - help="Name of the person or organization that converted the data.", - default=os.getenv("FIBOA_CONVERTED_BY"), + help="Additional options passed to tippecanoe.", + default=TIPPECANOE_DEFAULT_OPTS, show_default=True, ), } @@ -108,337 +71,174 @@ def callback(dataset, *args, **kwargs): return callback - def __init__(self, dataset: str, data_url=None, s3_upload_path=None): + def __init__(self, dataset: str): super().__init__() self.cmd_title = f"Publish {dataset}" self.dataset = dataset - self.data_url = data_url or f"{self.url_base}/{self.dataset}" - self.s3_upload_path = ( - s3_upload_path or f"s3://us-west-2.opendata.source.coop/fiboa/data/{self.dataset}/" - ) try: self.converter = Converters().load(self.dataset) except (ImportError, NameError, OSError, RuntimeError, SyntaxError) as e: raise Exception(f"Converter for '{self.dataset}' not available or faulty: {e}") from e - def exc(self, cmd): - assert os.system(cmd) == 0 - def check_command(self, cmd, name=None): - if os.system(f"{cmd} --version") != 0: + if shutil.which(cmd) is None: self.error(f"Missing command {cmd}. Please install {name or cmd}") sys.exit(1) - def download_data_survey(self, base, **kwargs): - data_survey = ( - kwargs.get("data_survey_url") - or f"https://raw.githubusercontent.com/fiboa/data-survey/refs/heads/main/data/{base}.md" - ) - response = requests.get(data_survey) - if not response.ok: - self.warning( - f"Missing data survey {base}.md at {data_survey}. Falling back to converter declared properties." - ) - else: - return response.text - - @cache - def collect_meta_data(self, parquet_file, **kwargs): - base = self.dataset.replace("_", "-").upper() - data = { - "provider": self.converter.provider, - "license": self.converter.license, - "projection": "", - "homepage": "", - "submitter": "Fiboa project", - "header": "", - } - text = self.download_data_survey(base, **kwargs) - mapping = { - "data provider (legal entity)": "provider", - "submitter (affiliation)": "submitter", - } - properties = {} - if text: - data["header"] = ( - f"\n- **Data Survey:** https://github.com/fiboa/data-survey/blob/main/data/{base}.md" - ) - data.update( - { - mapping.get(a.lower(), a.lower()): b - for a, b in re.findall(r"- \*\*(.+?):\*\* (.+?)\n", text) - } - ) - properties = { - a.lower(): b.strip() - for a, b in re.findall(r"\n\|\s*(\w+)[^|]*\|[^|]*\|[^|]*\|([^|]*)\|", text) - } - try: - # Try read projection from parquet metadata - meta = create_encoding(parquet_file).get_geoparquet_metadata() - crs = meta["columns"]["geometry"]["crs"] - data["projection"] = f"{crs['id']['authority']}:{crs['id']['code']} ({crs['name']})" - except Exception: - pass - converted_by = kwargs.get("converted_by") - if converted_by: - data["submitter"] = converted_by - - assert data["provider"], "Cannot determine data provider from converter or data survey." - return data, properties - - def readme_attribute_table(self, stac_data, properties): - def description(name): - m = self.converter.columns - reverse = dict(zip(m.values(), m.keys())) - return ( - properties.get(reverse.get(name)) - or properties.get(name) - or DESCRIPTIONS.get(name, "") - ) - - cols = [["Property", "**Data Type**", "Description"]] + [ - [ - s["name"], - re.search(r"\w+", s["type"])[0], - description(s["name"]), - ] - for s in stac_data["assets"]["data"]["table:columns"] - if s["name"] not in ("geometry", "bbox", "collection") - ] - widths = [max(len(c[i]) for c in cols) for i in range(3)] - aligned_cols = [[f" {c:<{w}} " for c, w in zip(row, widths)] for row in cols] - aligned_cols.insert(1, ["-" * (w + 2) for w in widths]) - return "\n".join(["|" + "|".join(cols) + "|" for cols in aligned_cols]) - - def make_license(self, parquet_file, **kwargs): - text = "" - try: - data, properties = self.collect_meta_data(parquet_file, **kwargs) - text = data["license"] - if getattr(self.converter, "license") not in (None, "", data["license"]): - text += "\n" + self.converter.license + "\n" - - found = False - for _license in (data["license"], self.converter.license): - if not _license or "<(https://" in _license: - continue - - # Include full-license text - _license = _license.upper() - if _license in spdx_license_list.LICENSES: - response = requests.get( - f"https://raw.githubusercontent.com/spdx/license-list-data/refs/heads/main/text/{_license}.txt" - ) - if response.ok: - found = True - text += f"\n\n{response.text}\n" - break - if not found: - self.warning(f"License {text} could not be found in SPDX license list") - - except Exception as e: - self.exception(e) - return text - - def make_readme(self, parquet_file, file_name, stac, **kwargs): - version = Registry.get_version() - converter = self.converter - with open(stac) as f: - stac_data = json.load(f) - count = stac_data["assets"]["data"]["table:row_count"] - data, properties = self.collect_meta_data(parquet_file, **kwargs) - columns = self.readme_attribute_table(stac_data, properties) - urls = converter.get_urls() or "manually downloaded file" - urls = urls.keys() if isinstance(urls, dict) else [urls] - downloaded_urls = "\n".join([(" - " + url) for url in urls]) - - return f"""# Field boundaries for {converter.short_name} - -Provides {count} official field boundaries from {converter.short_name}. -It has been converted to a fiboa GeoParquet file from data obtained from {data["provider"]}. - -- **Source Data Provider:** [{data["provider"]}]({data["homepage"]}) -- **Converted by:** {data["submitter"]} -- **License:** {data["license"]} -- **Projection:** {data["projection"]}{data["header"]} - ---- - -- [Download the data as fiboa GeoParquet]({self.data_url}/{file_name}.parquet) -- [STAC Browser](https://radiantearth.github.io/stac-browser/#/external/data.source.coop/fiboa/data/{self.dataset}/stac/collection.json) -- [STAC Collection]({self.data_url}/stac/collection.json) -- [PMTiles]({self.data_url}/{file_name}.pmtiles) - -## Columns - -{columns} - -## Lineage - -- Data downloaded on {date.today()} from: -{downloaded_urls} -- Converted to GeoParquet using [fiboa-cli](https://github.com/fiboa/cli), version {version} -""" - @runnable def publish( self, target, - generate_meta=False, - yes=False, - data_survey_url=None, - editor=None, - converted_by=None, + pmtiles=True, + tippecanoe_opts=TIPPECANOE_DEFAULT_OPTS, **kwargs, ): """ - You need GDAL 3.8 or later (for ogr2ogr) with libgdal-arrow-parquet, tippecanoe, and AWS CLI + Creates the following files in the target folder: + + - [-].parquet: the converted and validated fiboa GeoParquet file + - [-].pmtiles: vector tiles for visualization (ogr2ogr + tippecanoe) + - collection.json: a STAC Collection with relative links to the files above + + Existing files are reused, delete them to regenerate. + PMTiles generation needs GDAL 3.8 or later (for ogr2ogr) and tippecanoe: - https://gdal.org/ - https://github.com/felt/tippecanoe - - https://aws.amazon.com/cli/ """ - Path(target).mkdir(parents=True, exist_ok=True) + target = Path(target) + target.mkdir(parents=True, exist_ok=True) file_name = self.dataset - if not kwargs["variant"] and self.converter.variants: + if not kwargs.get("variant") and self.converter.variants: kwargs["variant"] = next(iter(self.converter.variants)) - if kwargs["variant"]: + if kwargs.get("variant"): file_name += f"-{kwargs['variant']}" - parquet_file = Path(target) / f"{file_name}.parquet" - - has_write_access = bool( - os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY") - ) + parquet_file = target / f"{file_name}.parquet" + pmtiles_file = target / f"{file_name}.pmtiles" + stac_file = target / "collection.json" - stac_file = Path(target) / "stac" / "collection.json" - - ## Create parquet file + # Create parquet file if not parquet_file.exists(): - self.info(f"Converting file for {self.dataset} to {parquet_file}") + self.info(f"Converting {self.dataset} to {parquet_file}") ConvertData(self.dataset).run(parquet_file, **kwargs) - self.success(f"Converted file for {self.dataset} to {parquet_file}") + self.success(f"Converted {self.dataset} to {parquet_file}") else: - self.success(f"Using existing file {parquet_file} for {self.dataset}") + self.success(f"Using existing file {parquet_file}") - ## Validate parquet file, we only want to publish valid files + # Validate parquet file, we only want to publish valid files self.info(f"Validating {parquet_file}") ValidateData().validate(parquet_file, num=-1) self.log("\n => VALID\n", "success") - ## Create STAC collection.json - self.create_stac_collection(target, file_name, parquet_file, stac_file) - - if generate_meta: - self.generate_meta( - target, - file_name, - stac_file, - data_survey_url=data_survey_url, - converted_by=converted_by, - yes=yes, - editor=editor, - ) - - self.generate_pmtiles(target, file_name, parquet_file) - if not has_write_access: - self.info("Get your credentials through the source coop organization.") - self.info("Login to AWS Console and generate an access key:") - self.info( - " - In AWS console, click on account (right top) press 'Security credentials'," - ) - self.info(" - Go to 'Access keys' and press 'Create access key'") - self.info( - " - Run `export AWS_ACCESS_KEY_ID=<> AWS_SECRET_ACCESS_KEY=<>`\n" - " (Linux/Mac only) where you copy-paste the access key and secret to <>.", - ) - self.error("Please set AWS_ environment variables for uploading") - return - self.upload_to_aws(target) - - def create_stac_collection(self, target, file_name, parquet_file, stac_file): - p_stac = Path(stac_file) - if p_stac.exists() and p_stac.stat().st_mtime >= Path(parquet_file).stat().st_mtime: + # Create PMTiles + if pmtiles: + self.generate_pmtiles(parquet_file, pmtiles_file, tippecanoe_opts) + has_pmtiles = pmtiles_file.exists() + + # Create STAC collection.json + self.create_stac_collection(parquet_file, pmtiles_file if has_pmtiles else None, stac_file) + self.success(f"Created {stac_file}") + return stac_file + + def create_stac_collection(self, parquet_file: Path, pmtiles_file, stac_file: Path): + is_current = ( + stac_file.exists() + and stac_file.stat().st_mtime >= parquet_file.stat().st_mtime + and (pmtiles_file is None or stac_file.stat().st_mtime >= pmtiles_file.stat().st_mtime) + ) + if is_current: + self.info(f"Reusing existing {stac_file}") return - self.success(f"Creating STAC collection.json for {parquet_file}") - p_stac.parent.mkdir(exist_ok=True) - CreateStacCollection().create_cli(parquet_file, stac_file) - - Path(target, "stac").mkdir(parents=True, exist_ok=True) - data = json.load(open(stac_file, "r")) - assert data["id"] == self.dataset, ( - f"Wrong collection dataset id: {data['id']} != {self.dataset}, for {stac_file}" + self.info(f"Creating STAC collection for {parquet_file}") + data = CreateStacCollection().create_from_file( + parquet_file, data_url=f"./{parquet_file.name}" ) + if data["id"] != self.dataset: + raise Exception( + f"Wrong collection id: {data['id']} != {self.dataset}, for {parquet_file}" + ) - data["assets"]["data"]["href"] = f"{self.data_url}/{file_name}.parquet" + extensions = data.setdefault("stac_extensions", []) + if FILE_EXTENSION not in extensions: + extensions.append(FILE_EXTENSION) - if STAC_EXTENSION not in data["stac_extensions"]: - data["stac_extensions"].append(STAC_EXTENSION) + asset = data["assets"]["data"] + asset["title"] = f"{data.get('title') or self.dataset} (GeoParquet)" + asset.update(self.file_metadata(parquet_file)) - if not any(d.get("rel") == "pmtiles" for d in data["links"]): + if pmtiles_file is not None: + if WEB_MAP_LINKS_EXTENSION not in extensions: + extensions.append(WEB_MAP_LINKS_EXTENSION) + data["links"] = [link for link in data.get("links", []) if link.get("rel") != "pmtiles"] data["links"].append( { - "href": f"{self.data_url}/{file_name}.pmtiles", - "type": "application/vnd.pmtiles", "rel": "pmtiles", + "href": f"./{pmtiles_file.name}", + "type": PMTILES_MEDIA_TYPE, + "title": "Web map tiles", + "pmtiles:layers": [self.dataset], } ) + data["assets"]["visual"] = { + "href": f"./{pmtiles_file.name}", + "type": PMTILES_MEDIA_TYPE, + "title": f"{data.get('title') or self.dataset} (PMTiles)", + "roles": ["visual"], + **self.file_metadata(pmtiles_file), + } - with open(stac_file, "w", encoding="utf-8") as f: + with stac_file.open("w", encoding="utf-8") as f: json.dump(data, f, indent=2) - def generate_meta(self, target, file_name, stac_file, **kwargs): - parquet_file = Path(target) / f"{file_name}.parquet" - for required in ("README.md", "LICENSE.txt"): - path = Path(target) / required - if not path.exists(): - self.warning(f"Missing {required}. Generating at {path}") - if required == "README.md": - text = self.make_readme( - parquet_file, - file_name=file_name, - stac=stac_file, - **kwargs, - ) - else: - text = self.make_license(parquet_file, **kwargs) - self.info( - f"\nGenerated the following file {required}:\n{'-' * 80}\n\n{text}\n{'-' * 80}\n" - ) - action = ( - "C" - if kwargs.get("yes") - else input("Do you want to Continue (C), Edit (E) or Abort (A)?") - ) - if action.lower() not in "ce": - self.warning("Bailing out") - sys.exit(1) - with open(path, "w") as f: - f.write(text) - editor = kwargs.get("editor") - if action.lower() == "e" and editor: - os.system(f"{editor} {path}") + @staticmethod + def file_metadata(path: Path) -> dict: + return { + "file:size": path.stat().st_size, + "file:checksum": multihash_sha256(path), + } - def generate_pmtiles(self, target, file_name, parquet_file): + def generate_pmtiles(self, parquet_file: Path, pmtiles_file: Path, tippecanoe_opts: str): if is_windows: self.warning( "PMTiles generation through tippecanoe is not supported on Windows, skipping." ) return + if pmtiles_file.exists(): + self.success(f"Using existing file {pmtiles_file}") + return - pm_file = Path(target) / f"{file_name}.pmtiles" - if not pm_file.exists(): - self.info("Running ogr2ogr | tippecanoe") - self.check_command("tippecanoe") - self.check_command("ogr2ogr", name="GDAL") - self.exc( - f"ogr2ogr -t_srs EPSG:4326 -f geojson /vsistdout/ {str(parquet_file)} | tippecanoe -zg --projection=EPSG:4326 -o {str(pm_file)} -l {self.dataset} --drop-densest-as-needed" - ) - - def upload_to_aws(self, target): - self.info("Uploading to aws") - - self.check_command("aws") - self.exc(f"aws s3 sync --exclude '.*' {target} {self.s3_upload_path}") + self.check_command("tippecanoe") + self.check_command("ogr2ogr", name="GDAL") + self.info("Running ogr2ogr | tippecanoe") + ogr = subprocess.Popen( + [ + "ogr2ogr", + "-t_srs", + "EPSG:4326", + "-f", + "GeoJSONSeq", + "/vsistdout/", + str(parquet_file), + ], + stdout=subprocess.PIPE, + ) + tippecanoe = subprocess.run( + [ + "tippecanoe", + *tippecanoe_opts.split(), + "--projection=EPSG:4326", + "-o", + str(pmtiles_file), + "-l", + self.dataset, + ], + stdin=ogr.stdout, + ) + ogr.stdout.close() + ogr.wait() + if ogr.returncode != 0 or tippecanoe.returncode != 0: + pmtiles_file.unlink(missing_ok=True) + raise Exception("PMTiles generation failed, see output above.") + self.success(f"Created {pmtiles_file}") diff --git a/pixi.lock b/pixi.lock index 1f6b2a69..9689ee5e 100644 --- a/pixi.lock +++ b/pixi.lock @@ -804,7 +804,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -940,7 +939,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/c4/3ce4c2d9b6aabd27d26ec988f08cb877ba9e6e96086eff81bfea93e688c7/shapely-2.1.2-cp314-cp314-macosx_10_13_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1077,7 +1075,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/b9/f6ab8918fc15429f79cb04afa9f9913546212d7fb5e5196132a2af46676b/shapely-2.1.2-cp314-cp314-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1213,7 +1210,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl - - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1327,7 +1323,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1426,7 +1421,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/c4/3ce4c2d9b6aabd27d26ec988f08cb877ba9e6e96086eff81bfea93e688c7/shapely-2.1.2-cp314-cp314-macosx_10_13_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1526,7 +1520,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/b9/f6ab8918fc15429f79cb04afa9f9913546212d7fb5e5196132a2af46676b/shapely-2.1.2-cp314-cp314-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1623,7 +1616,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl - - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -2803,10 +2795,9 @@ packages: - pypi: ./ name: fiboa-cli version: 0.21.0 - sha256: 5a8a4c3d9234870ac9a54c0cc925c41f59ca033ee111dd241c1af9633591f47b + sha256: 5b0a002f53b5ef4abfb8c3f5f429dc374fa4a4c9e0ef0fe34f6e13e3cce989da requires_dist: - vecorel-cli==0.2.15 - - spdx-license-list==3.27.0 requires_python: '>=3.11' editable: true - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda diff --git a/pyproject.toml b/pyproject.toml index 735559ce..66099f37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ classifiers = [ requires-python = ">=3.11" dependencies = [ "vecorel-cli==0.2.15", - "spdx-license-list==3.27.0", ] [project.scripts] @@ -62,6 +61,7 @@ ruff = "==0.12.8" [tool.pixi.feature.dev.pypi-dependencies] build = ">=1.0.0,<2.0.0" +spdx-license-list = "==3.27.0" [tool.pixi.feature.cloud.dependencies] s3fs = "==2025.7.0" diff --git a/tests/data-files/publish/BE-VLG-survey.md b/tests/data-files/publish/BE-VLG-survey.md deleted file mode 100644 index 9ac80ea3..00000000 --- a/tests/data-files/publish/BE-VLG-survey.md +++ /dev/null @@ -1,78 +0,0 @@ -# Vlaanderen, Belgium - -## Submission Details - -- **Submitter (Affiliation):** Matthias Mohr -- **Data Provider (Legal Entity):** Agriculture and Marine Fisheries Agency of the Flemish government (Government) -- **Homepage:** https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen -- **Alternative URL:** https://www.vlaanderen.be/datavindplaats/catalogus/landbouwgebruikspercelen-lv-2022 - -## Overview - -Since 2020, the Department of Agriculture and Fisheries has been publishing a more extensive set of data related to agricultural use plots (from the 2008 campaign). - -From 2023, the downloadable dataset of agricultural use plots will also include the specialization given by the company (= company typology) and that is given to the plots of the company. Based on the typology, the companies are divided into 4 major specializations: arable farming, horticulture, livestock farming and mixed farms. The specialization of each company is calculated annually according to a European method and is based on the standard output of the various agricultural productions on the company. It is therefore an economic specialization and not a reflection of all agricultural production on the company. - -## Data & Metadata - -- **URL:** https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen -- **Documentation:** contained in the ZIP packages -- **File Format:** GeoPackage / Shapefile -- **Projection:** EPSG:31370 (Belgian Lambert 72) -- **License:** CC-0 (described as "Publiek" and "Toegang zonder voorwaarden") - -### Properties - -Some of the documented fields are missing in the GeoPackage. These are marked with "(missing)". - -| Property | **Data Type** | Constraints | Description | -|-----------------------|---------------|----------------------------|----------------------------------------------------------------------------------------------| -| fid | integer | | Identifier | -| BT_OMSCH | string | 200 chars | Business type (economic specialization) | -| BT_BRON | string | 50 chars | Source of the business type (year of calculation or specialization indicated) | -| GRAF_OPP | number | | Area (ha, accurate to 1m²) | -| REF_ID | integer | | Unique identification number for the field. | -| GWSCOD_V | string | 5 chars (digits), nullable | Pre-cultivation code | -| GWSNAM_V | string | 90 chars, nullable | Pre-cultivation name | -| GWSCOD_H | string | 5 chars (digits), nullable | Main cultivation/crop code | -| GWSNAM_H | string | 90 chars, nullable | Main cultivation/crop name | -| GWSGRPH_LB | string | 150 chars, nullable | Main cultivation/crop group name | -| GWSCOD_N | string | 5 chars (digits), nullable | First cultivation/crop code | -| CWSNAM_N | string | 90 chars, nullable | First cultivation/crop name | -| GWSCOD_N2 | string | 5 chars (digits), nullable | Second cultivation/crop code | -| GWSNAM_N2 | string | 90 chars, nullable | Second cultivation/crop name | -| AMKM (missing) | string | | Agri-environment code | -| AMKM_LB (missing) | string | | Agri-environment name | -| ECOREGELING (missing) | string | | Eco-regulation code | -| ECOR_LB (missing) | string | | Eco-regulation name | -| BLS (missing) | string | | Planting subsidy code (forest farming systems) | -| BLS_LB (missing) | string | | Planting subsidy name (forest farming systems) | -| GESP_PM | string | 11 chars, nullable | Specialized production method | -| GESP_PM_LB | string | 150 chars, nullable | Description of specialized production method | -| BIOCERT (missing) | string | `J` or `N` | Plot under bio-control with a bio-control body. | -| ERO_NAM | string | 20 chars, | Erosion color code for the field | -| STAT_BGV | string | 2 chars, nullable | Status Permanent Grassland under greening (BG) | -| MEERJARIG_GRASLAND | string | | Status Perennial Grassland (MG6 or higher). Example: MG16 = 16th year grassland | -| LANDBSTR | string | 2 chars, nullable | Agricultural region in which the center of the field is located | -| STAT_AAR | string | 10 chars, nullable | Status Potatoes, follow up rotation duty | -| PCT_EKBG | string | 10 chars, nullable | Percentage range of field that is ecologically sensitive permanent pasture. Example: `0-10%` | -| PCT_WETVEEN | string | 10 chars, nullable | Percentage range of field that is wetland and/or peatland. Example: `0-10%` | -| PRC_GEM | string | 30 chars | Municipality in which the center of the field is located | -| PRC_NIS | string | 5 chars (digits) | NIS code of the municipality in which the center of the field is located | -| X_REF | number | | X coordinate of the center of the field (Lambert) | -| Y_REF | number | | Y coordinate of the center of the field (Lambert) | -| WGS84_LG | string | 11 chars | Longitude of the center of the field (WGS84). Example: `3°21'44"` | -| WGS84_BG | string | 11 chars | Latitude of the center of the field (WGS84). Example: `51°11'39"` | - -Note: Many integer-like numbers are encoded as strings. - -## API - -The open data viewer https://geopunt.be/ shows the data in a viewer (search term: landbouwgebruikspercelen) -See https://www.vlaanderen.be/datavindplaats/catalogus/landbouwgebruikspercelen-lv-2022 for more info - -| Standard | URL | Documentation | -|--------------|-------------------------------------------------------------|------------------------------------------------------------------------------------------------------| -| OGC WFS | https://geo.api.vlaanderen.be/Landbgebrperc/wfs | https://www.vlaanderen.be/datavindplaats/catalogus/wfs-landbouwgebruikspercelen | -| OGC Features | https://geo.api.vlaanderen.be/Landbgebrperc/ogc/features/v1 | https://metadata.vlaanderen.be/srv/dut/catalog.search#/metadata/01f408db-df8a-49a2-8ce4-0f66b8efe17b | -| OGC WMS | https://geo.api.vlaanderen.be/ALV/wms | https://www.vlaanderen.be/datavindplaats/catalogus/wms-departement-landbouw-en-visserij | diff --git a/tests/test_publish.py b/tests/test_publish.py index cf960760..021cb5ef 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -1,30 +1,43 @@ -import responses +import json from fiboa_cli.publish import Publish class PublishTest(Publish): - def generate_pmtiles(self, target, file_name, parquet_file): - pass + def generate_pmtiles(self, parquet_file, pmtiles_file, tippecanoe_opts): + # tippecanoe is not available everywhere, fake the tiles + pmtiles_file.write_bytes(b"PMTiles") - def upload_to_aws(self, target): - pass - -@responses.activate def test_publish(tmp_folder): converter = "be_vlg" - base = "BE-VLG" path = f"tests/data-files/convert/{converter}" - rsp1 = responses.Response( - method="GET", - url=f"https://raw.githubusercontent.com/fiboa/data-survey/refs/heads/main/data/{base}.md", - body=open(f"tests/data-files/publish/{base}-survey.md").read(), - ) - responses.add(rsp1) - PublishTest(converter).run( - variant="2023", target=tmp_folder, cache=path, generate_meta=True, yes=True - ) + PublishTest(converter).run(variant="2023", target=tmp_folder, cache=path) + files = [f.name for f in tmp_folder.iterdir() if f.is_file()] - for f in ("README.md", "LICENSE.txt", "be_vlg-2023.parquet"): + for f in ("collection.json", "be_vlg-2023.parquet", "be_vlg-2023.pmtiles"): assert f in files, f"Missing file {f}" + + with open(tmp_folder / "collection.json") as f: + stac = json.load(f) + + assert stac["id"] == converter + assert "https://stac-extensions.github.io/file/v2.1.0/schema.json" in stac["stac_extensions"] + assert ( + "https://stac-extensions.github.io/web-map-links/v1.3.0/schema.json" + in stac["stac_extensions"] + ) + + data = stac["assets"]["data"] + assert data["href"] == "./be_vlg-2023.parquet" + assert data["file:size"] == (tmp_folder / "be_vlg-2023.parquet").stat().st_size + assert data["file:checksum"].startswith("1220") and len(data["file:checksum"]) == 68 + + visual = stac["assets"]["visual"] + assert visual["href"] == "./be_vlg-2023.pmtiles" + assert visual["roles"] == ["visual"] + assert visual["file:size"] == 7 + + pmtiles = next(link for link in stac["links"] if link["rel"] == "pmtiles") + assert pmtiles["href"] == "./be_vlg-2023.pmtiles" + assert pmtiles["pmtiles:layers"] == [converter] From f8944b431f9534bc76e0626d32fb4603de6f9d71 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 22:33:25 +0200 Subject: [PATCH 02/13] NL: new PDOK download location, 2026 concept edition; DE-TH: INSPIRE note Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 2 ++ fiboa_cli/datasets/de_th.py | 2 ++ fiboa_cli/datasets/nl.py | 10 ++++++---- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a80ad1e..986925f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) +- NL: new PDOK download location (rvo/gewaspercelen/atom), add the 2026 concept edition +- DE-TH: note the INSPIRE download service - BE-VLG: drop plots without a crop code (one such plot in the 2023 edition made validation fail); derive determination:datetime from the variant year instead of a constant date - `fiboa publish` no longer uploads to S3 or generates README/LICENSE files. It creates GeoParquet, PMTiles and a STAC Collection with relative links, `file:size`/`file:checksum` and a web-map-links v1.3.0 `pmtiles` link. Publishing is done by catalogs such as the [harmonized field data catalog](https://github.com/fieldsoftheworld/harmonized-field-data-catalog). - Add Italy Tuscany (IT-1) basd on EuroCrops v2 diff --git a/fiboa_cli/datasets/de_th.py b/fiboa_cli/datasets/de_th.py index d272b1d9..215bda13 100644 --- a/fiboa_cli/datasets/de_th.py +++ b/fiboa_cli/datasets/de_th.py @@ -8,6 +8,8 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): sources = "https://www.geoproxy.geoportal-th.de/download-service/opendata/agrar/DGK_Thue.zip" + # https://www.geoproxy.geoportal-th.de/inspire-dl/ + # http://www.geoproxy.geoportal-th.de/inspire-dl/atom/DataSet/DataSet_06cd3e2f-ed4a-4507-b5e7-14973d4d6968.xml id = "de_th" admin_subdivision_code = "TH" diff --git a/fiboa_cli/datasets/nl.py b/fiboa_cli/datasets/nl.py index 625106ba..5388b7d0 100644 --- a/fiboa_cli/datasets/nl.py +++ b/fiboa_cli/datasets/nl.py @@ -4,13 +4,15 @@ from ..conversion.fiboa_converter import FiboaBaseConverter from .commons.hcat import AddHCATMixin -# see https://service.pdok.nl/rvo/brpgewaspercelen/atom/v1_0/basisregistratie_gewaspercelen_brp.xml -base = "https://service.pdok.nl/rvo/brpgewaspercelen/atom/v1_0/downloads" +# see https://service.pdok.nl/rvo/gewaspercelen/atom/basisregistratie_gewaspercelen_brp.xml +# (the old feed rvo/brpgewaspercelen/atom/v1_0/ redirects here since 2026) +base = "https://service.pdok.nl/rvo/gewaspercelen/atom/downloads" class NLCropConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): area_calculate_missing = True variants = { + "2026": f"{base}/gewaspercelen_concept_2026.gpkg", **{str(y): f"{base}/brpgewaspercelen_definitief_{y}.gpkg" for y in range(2025, 2020, -1)}, **{str(y): f"{base}/brpgewaspercelen_definitief_{y}.zip" for y in range(2020, 2009, -1)}, } @@ -27,9 +29,9 @@ class NLCropConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): A dataset is generated for each year with reference date May 15. A view service and a download service are available for the most recent BRP crop plots. - + -Data is currently available for the years 2009 to 2024. +Data is currently available for the years 2009 to 2025 (final) and 2026 (concept). """ provider = ( From 3b8d8f1cff7552b8f43cfffe8d66976cb4238b74 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 22:38:28 +0200 Subject: [PATCH 03/13] Declare beautifulsoup4 (used by es_pv, es_vc) Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + pixi.lock | 53 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 986925f5..54f16710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) +- Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import - NL: new PDOK download location (rvo/gewaspercelen/atom), add the 2026 concept edition - DE-TH: note the INSPIRE download service - BE-VLG: drop plots without a crop code (one such plot in the 2023 edition made validation fail); derive determination:datetime from the variant year instead of a constant date diff --git a/pixi.lock b/pixi.lock index 9689ee5e..e870f765 100644 --- a/pixi.lock +++ b/pixi.lock @@ -140,6 +140,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl @@ -169,6 +170,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl @@ -301,6 +303,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h3eecb57_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl @@ -330,6 +333,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/c4/3ce4c2d9b6aabd27d26ec988f08cb877ba9e6e96086eff81bfea93e688c7/shapely-2.1.2-cp314-cp314-macosx_10_13_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl @@ -463,6 +467,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl @@ -492,6 +497,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/b9/f6ab8918fc15429f79cb04afa9f9913546212d7fb5e5196132a2af46676b/shapely-2.1.2-cp314-cp314-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl @@ -623,6 +629,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl @@ -652,6 +659,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl @@ -777,6 +785,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/90/72/0b6035302e9c33f004240a50cb6e2e1fc7bb1f2b415b02d939c551bdd06b/inflate64-1.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -804,6 +813,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -912,6 +922,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h3eecb57_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/80/24ba0d2ee14e07e275e9c5b058e59a8a58f8ef42dd51a78ebbfd7c857ac4/inflate64-1.0.4-cp314-cp314-macosx_10_15_x86_64.whl @@ -939,6 +950,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/c4/3ce4c2d9b6aabd27d26ec988f08cb877ba9e6e96086eff81bfea93e688c7/shapely-2.1.2-cp314-cp314-macosx_10_13_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1048,6 +1060,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/b8/073a79716e093db973b8823bdfb02e10fbdf65642dbe1fa3cda24832aeb2/inflate64-1.0.4-cp314-cp314-macosx_11_0_arm64.whl @@ -1075,6 +1088,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/b9/f6ab8918fc15429f79cb04afa9f9913546212d7fb5e5196132a2af46676b/shapely-2.1.2-cp314-cp314-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1183,6 +1197,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/51/f2972df8cceecc9bf3afa3353d517ffc7125285198c844588e9aaf98f5d0/inflate64-1.0.4-cp314-cp314-win_amd64.whl @@ -1210,6 +1225,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1296,6 +1312,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/90/72/0b6035302e9c33f004240a50cb6e2e1fc7bb1f2b415b02d939c551bdd06b/inflate64-1.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -1323,6 +1340,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1394,6 +1412,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h3eecb57_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/80/24ba0d2ee14e07e275e9c5b058e59a8a58f8ef42dd51a78ebbfd7c857ac4/inflate64-1.0.4-cp314-cp314-macosx_10_15_x86_64.whl @@ -1421,6 +1440,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/c4/3ce4c2d9b6aabd27d26ec988f08cb877ba9e6e96086eff81bfea93e688c7/shapely-2.1.2-cp314-cp314-macosx_10_13_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1493,6 +1513,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/b8/073a79716e093db973b8823bdfb02e10fbdf65642dbe1fa3cda24832aeb2/inflate64-1.0.4-cp314-cp314-macosx_11_0_arm64.whl @@ -1520,6 +1541,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/b9/f6ab8918fc15429f79cb04afa9f9913546212d7fb5e5196132a2af46676b/shapely-2.1.2-cp314-cp314-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1589,6 +1611,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/51/f2972df8cceecc9bf3afa3353d517ffc7125285198c844588e9aaf98f5d0/inflate64-1.0.4-cp314-cp314-win_amd64.whl @@ -1616,6 +1639,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl @@ -1727,6 +1751,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl @@ -1756,6 +1781,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl @@ -1853,6 +1879,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h3eecb57_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl @@ -1882,6 +1909,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/c4/3ce4c2d9b6aabd27d26ec988f08cb877ba9e6e96086eff81bfea93e688c7/shapely-2.1.2-cp314-cp314-macosx_10_13_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl @@ -1980,6 +2008,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl @@ -2009,6 +2038,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/b9/f6ab8918fc15429f79cb04afa9f9913546212d7fb5e5196132a2af46676b/shapely-2.1.2-cp314-cp314-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl @@ -2103,6 +2133,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl @@ -2132,6 +2163,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl @@ -2285,6 +2317,19 @@ packages: purls: [] size: 7514 timestamp: 1767044983590 +- pypi: https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl + name: beautifulsoup4 + version: 4.15.0 + sha256: d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9 + requires_dist: + - soupsieve>=1.6.1 + - typing-extensions>=4.0.0 + - cchardet ; extra == 'cchardet' + - chardet ; extra == 'chardet' + - charset-normalizer ; extra == 'charset-normalizer' + - html5lib ; extra == 'html5lib' + - lxml ; extra == 'lxml' + requires_python: '>=3.7.0' - conda: https://conda.anaconda.org/conda-forge/noarch/blinker-1.9.0-pyhff2d567_0.conda sha256: f7efd22b5c15b400ed84a996d777b6327e5c402e79e3c534a7e086236f1eb2dc md5: 42834439227a4551b939beeeb8a4b085 @@ -2795,9 +2840,10 @@ packages: - pypi: ./ name: fiboa-cli version: 0.21.0 - sha256: 5b0a002f53b5ef4abfb8c3f5f429dc374fa4a4c9e0ef0fe34f6e13e3cce989da + sha256: 15c4c23b3c81265e8c7ee326e01ae3f44762998d0b463a66752ce2f5683b10d2 requires_dist: - vecorel-cli==0.2.15 + - beautifulsoup4>=4.12 requires_python: '>=3.11' editable: true - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda @@ -6724,6 +6770,11 @@ packages: - pkg:pypi/six?source=hash-mapping size: 18455 timestamp: 1753199211006 +- pypi: https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl + name: soupsieve + version: 2.9.2 + sha256: 8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/6e/d5/6fbc5770fc55e027dbd24571c4fd0b4ad6f2e310adbbda95ec39993f344c/spdx_license_list-3.27.0-py3-none-any.whl name: spdx-license-list version: 3.27.0 diff --git a/pyproject.toml b/pyproject.toml index 66099f37..19c52c92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ requires-python = ">=3.11" dependencies = [ "vecorel-cli==0.2.15", + "beautifulsoup4>=4.12", ] [project.scripts] From d22b6030530d6382faac4ccce520e74a539a83a8 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 22:44:57 +0200 Subject: [PATCH 04/13] Drop rows without a required crop:code instead of failing the conversion Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 3 ++- fiboa_cli/conversion/fiboa_converter.py | 10 ++++++++++ fiboa_cli/datasets/be_vlg.py | 5 ----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54f16710..39ddecfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import - NL: new PDOK download location (rvo/gewaspercelen/atom), add the 2026 concept edition - DE-TH: note the INSPIRE download service -- BE-VLG: drop plots without a crop code (one such plot in the 2023 edition made validation fail); derive determination:datetime from the variant year instead of a constant date +- Drop rows without a crop:code (required by the crop extension) with a warning instead of failing the conversion (BE-VLG 2023, ES-CN had one such row each) +- BE-VLG: derive determination:datetime from the variant year instead of a constant date - `fiboa publish` no longer uploads to S3 or generates README/LICENSE files. It creates GeoParquet, PMTiles and a STAC Collection with relative links, `file:size`/`file:checksum` and a web-map-links v1.3.0 `pmtiles` link. Publishing is done by catalogs such as the [harmonized field data catalog](https://github.com/fieldsoftheworld/harmonized-field-data-catalog). - Add Italy Tuscany (IT-1) basd on EuroCrops v2 - Suuport multiple years for CZ diff --git a/fiboa_cli/conversion/fiboa_converter.py b/fiboa_cli/conversion/fiboa_converter.py index a073b0e5..53aa2e54 100644 --- a/fiboa_cli/conversion/fiboa_converter.py +++ b/fiboa_cli/conversion/fiboa_converter.py @@ -4,6 +4,9 @@ from ..fiboa.version import get_fiboa_uri AREA_KEY = "metrics:area" +# Properties that a schema requires to be non-null; rows lacking them cannot +# validate, so they are dropped (with a warning) rather than failing the run. +REQUIRED_NON_NULL = ("crop:code",) class FiboaBaseConverter(BaseConverter): @@ -22,6 +25,13 @@ def __init__(self, *args, **kwargs): def post_migrate(self, gdf): gdf = super().post_migrate(gdf) + for key in REQUIRED_NON_NULL: + if key in gdf.columns: + nulls = gdf[key].isna() + if nulls.any(): + self.warning(f"Dropping {int(nulls.sum())} rows without a value for {key}") + gdf = gdf[~nulls] + gdf_area_key = next((k for k, v in self.columns.items() if v == AREA_KEY), None) if self.area_calculate_missing: # If CRS is not in meters, reproject to an equal-area projection for area calculation diff --git a/fiboa_cli/datasets/be_vlg.py b/fiboa_cli/datasets/be_vlg.py index b8dc6d9a..c33e2282 100644 --- a/fiboa_cli/datasets/be_vlg.py +++ b/fiboa_cli/datasets/be_vlg.py @@ -46,11 +46,6 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): # Each edition is the campaign year of its variant; the old constant # "2024-03-28" was the extraction date of one edition applied to all of them. use_variant_as_determination = True - column_filters = { - # A handful of plots (e.g. one in 2023, typology "Niet-geclassificeerd") carry no - # crop code; crop:code is required by the crop extension, so drop them. - "GWSCOD_H": lambda col: col.notna(), - } ec_mapping_csv = "be_vlg_2021.csv" missing_schemas = {"properties": {"typology": {"type": "string"}}} From b92251a8cbb7d6e5ec435de917936b00f8ffbaca Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 22:46:25 +0200 Subject: [PATCH 05/13] Look up the source column when dropping rows without a crop:code Co-Authored-By: Claude Fable 5 --- fiboa_cli/conversion/fiboa_converter.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/fiboa_cli/conversion/fiboa_converter.py b/fiboa_cli/conversion/fiboa_converter.py index 53aa2e54..0ff5cfd5 100644 --- a/fiboa_cli/conversion/fiboa_converter.py +++ b/fiboa_cli/conversion/fiboa_converter.py @@ -25,12 +25,17 @@ def __init__(self, *args, **kwargs): def post_migrate(self, gdf): gdf = super().post_migrate(gdf) + # post_migrate runs before columns are renamed, so look up the source column for key in REQUIRED_NON_NULL: - if key in gdf.columns: - nulls = gdf[key].isna() - if nulls.any(): - self.warning(f"Dropping {int(nulls.sum())} rows without a value for {key}") - gdf = gdf[~nulls] + for src, dst in self.columns.items(): + targets = dst if isinstance(dst, (list, tuple)) else [dst] + if key in targets and src in gdf.columns: + nulls = gdf[src].isna() + if nulls.any(): + self.warning( + f"Dropping {int(nulls.sum())} rows without a value for {key} ({src})" + ) + gdf = gdf[~nulls] gdf_area_key = next((k for k, v in self.columns.items() if v == AREA_KEY), None) if self.area_calculate_missing: From a4afe8801f28757c811a5ef9ed15b1b79ae4fb8c Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 22:47:48 +0200 Subject: [PATCH 06/13] DE-BB: read the shapefile as cp1252 Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + fiboa_cli/datasets/de_bb.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39ddecfa..3ce2f587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import +- DE-BB: read the shapefile as cp1252 (its .cpg wrongly says UTF-8) - NL: new PDOK download location (rvo/gewaspercelen/atom), add the 2026 concept edition - DE-TH: note the INSPIRE download service - Drop rows without a crop:code (required by the crop extension) with a warning instead of failing the conversion (BE-VLG 2023, ES-CN had one such row each) diff --git a/fiboa_cli/datasets/de_bb.py b/fiboa_cli/datasets/de_bb.py index 2294bcc0..c25d734d 100644 --- a/fiboa_cli/datasets/de_bb.py +++ b/fiboa_cli/datasets/de_bb.py @@ -14,6 +14,8 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): license = "DL-DE-BY-2.0" provider = "Land Brandenburg " ec_mapping_csv = "de.csv" + # The .cpg claims UTF-8 but the DBF is cp1252 (June 2026 download) + open_options = dict(encoding="cp1252") columns = { "geometry": "geometry", From 53204af984a6960d5714110cf3409c2cc957d397 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 23:05:57 +0200 Subject: [PATCH 07/13] CZ: find the shapefile in nested archive folders Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + fiboa_cli/datasets/cz.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ce2f587..6817e622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import +- CZ: find the shapefile in nested archive folders (2026) - DE-BB: read the shapefile as cp1252 (its .cpg wrongly says UTF-8) - NL: new PDOK download location (rvo/gewaspercelen/atom), add the 2026 concept edition - DE-TH: note the INSPIRE download service diff --git a/fiboa_cli/datasets/cz.py b/fiboa_cli/datasets/cz.py index e02f7f69..ad04aafb 100644 --- a/fiboa_cli/datasets/cz.py +++ b/fiboa_cli/datasets/cz.py @@ -20,7 +20,8 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): # see https://mze.gov.cz/public/app/eagriapp/lpisdata/ - variants = {str(k): {BASE.format(v): ["*.shp"]} for k, v in ITEMS.items()} + # the 2026 archive nests the shapefile in a folder, older ones are flat + variants = {str(k): {BASE.format(v): ["**/*.shp"]} for k, v in ITEMS.items()} id = "cz" short_name = "Czech" title = "Field boundaries for Czech" From 33d5ffd6b03b453eaa682c1d796507dc9b56ca85 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 23:11:58 +0200 Subject: [PATCH 08/13] ES-CAT: the 2024 download is a shapefile package Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + fiboa_cli/datasets/es_cat.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6817e622..ca263053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import +- ES-CAT: the 2024 download is a shapefile package, not a GeoPackage - CZ: find the shapefile in nested archive folders (2026) - DE-BB: read the shapefile as cp1252 (its .cpg wrongly says UTF-8) - NL: new PDOK download location (rvo/gewaspercelen/atom), add the 2026 concept edition diff --git a/fiboa_cli/datasets/es_cat.py b/fiboa_cli/datasets/es_cat.py index 1987bcfe..11fa36c5 100644 --- a/fiboa_cli/datasets/es_cat.py +++ b/fiboa_cli/datasets/es_cat.py @@ -9,7 +9,7 @@ class ESCatConverter(FiboaBaseConverter): variants = { "2024": { "https://analisi.transparenciacatalunya.cat/api/views/yh94-j2n9/files/d90f5fca-ddd8-405d-a0d5-90609985e98e?download=true&filename=Cultius_DUN2024_SHP.zip": [ - "Cultius_DUN2024_GPKG/CULTIUS_DUN2024.gpkg" + "Cultius_DUN2024_SHP/Cultius_DUN2024_SHP.shp" ] }, "2023": { @@ -66,7 +66,7 @@ def layer_filter(self, layer, uri): def migrate(self, gdf): # In 2023 gpkg, names are lowercase. But in 2022 shapefile, case is mixed - to_lower = {k: k.lower() for k in gdf.columns if k != k.lower} + to_lower = {k: k.lower() for k in gdf.columns if k != k.lower()} if to_lower: gdf.rename(columns=to_lower, inplace=True) From 4eeff3033c416d82d4af4f9b9d854e660c07eef6 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 23:13:39 +0200 Subject: [PATCH 09/13] SK: row index as id, KODKD block code as block_id Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + fiboa_cli/datasets/sk.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca263053..b9a9991f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import +- SK: KODKD is the (non-unique, sometimes empty) LPIS block code, keep it as block_id and use the row index as id - ES-CAT: the 2024 download is a shapefile package, not a GeoPackage - CZ: find the shapefile in nested archive folders (2026) - DE-BB: read the shapefile as cp1252 (its .cpg wrongly says UTF-8) diff --git a/fiboa_cli/datasets/sk.py b/fiboa_cli/datasets/sk.py index 71b78891..6fa84b3e 100644 --- a/fiboa_cli/datasets/sk.py +++ b/fiboa_cli/datasets/sk.py @@ -27,9 +27,12 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): provider = "Pôdohospodárska platobná agentúra " license = "CC0-1.0" # "Open Data" ec_mapping_csv = "https://fiboa.org/code/sk/sk.csv" + # KODKD is the LPIS block code, shared by several fields and sometimes empty; + # the row index is the field id and the code is kept as block_id. + index_as_id = True columns = { "geometry": "geometry", - "KODKD": "id", + "KODKD": "block_id", "PLODINA": "crop:name", "KULTURA_NA": "crop_group", "LOKALITA_N": "municipality", @@ -37,6 +40,7 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): } missing_schemas = { "properties": { + "block_id": {"type": "string"}, "crop_group": {"type": "string"}, "municipality": {"type": "string"}, } From 077238eb40cbb42d3d076ec77bd0ef8ae988756d Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 23:14:15 +0200 Subject: [PATCH 10/13] ES-CAT: map the 34 crop names new in the 2024 edition Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 2 +- fiboa_cli/datasets/data-files/es_cat.csv | 34 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9a9991f..c89da358 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import - SK: KODKD is the (non-unique, sometimes empty) LPIS block code, keep it as block_id and use the row index as id -- ES-CAT: the 2024 download is a shapefile package, not a GeoPackage +- ES-CAT: the 2024 download is a shapefile package, not a GeoPackage; 34 crop names new in 2024 added to the mapping - CZ: find the shapefile in nested archive folders (2026) - DE-BB: read the shapefile as cp1252 (its .cpg wrongly says UTF-8) - NL: new PDOK download location (rvo/gewaspercelen/atom), add the 2026 concept edition diff --git a/fiboa_cli/datasets/data-files/es_cat.csv b/fiboa_cli/datasets/data-files/es_cat.csv index c2b8225c..435cc08b 100644 --- a/fiboa_cli/datasets/data-files/es_cat.csv +++ b/fiboa_cli/datasets/data-files/es_cat.csv @@ -297,3 +297,37 @@ original_code,original_name,translated_name 310,MANDARINER,MANDARIN 311,'ALGARROBA' HERBÀCIA,ALGARROBA 312,ESPÍGOL O LAVANDA,LAVENDER +313,ALOE VERA,ALOE VERA +314,ANTARA VINYA,VINEYARD INTER-ROW +315,"ARANYONER, PRUNYONER",BLACKTHORN (SLOE) +316,"BITXO, VITXO",CHILI PEPPER +317,CIBULET,CHIVES +318,CIVADA I BLAT,OATS AND WHEAT +319,CIVADA I ORDI,OATS AND BARLEY +320,CIVADA I TRITICALE,OATS AND TRITICALE +321,COL KALE,KALE +322,CROTALÀRIA,CROTALARIA (SUNN HEMP) +323,ERBS I CIVADA,BITTER VETCH AND OATS +324,ESPELTA PETITA,EINKORN +325,GINJOLER,JUJUBE +326,LAVANDA X ESPIGOL,LAVANDIN +327,LOT CORNICULAT,BIRD'S-FOOT TREFOIL +328,MALVA,MALLOW +329,MARIALLUÏSA,LEMON VERBENA +330,"MARXANT, BLET",AMARANTH +331,"MENTA VERDA, HERBA DE SANTA MARIA",SPEARMINT +332,"MILL ITALIÀ, CUA DE GUILLA",FOXTAIL MILLET +333,MONGETA VERMELLA,RED KIDNEY BEAN +334,MORER,MULBERRY +335,"NYÀMERA, PATATA DE CANYA",JERUSALEM ARTICHOKE +336,OKRA,OKRA +337,PASSACAMINS,KNOTGRASS +338,"PISANA, ESPELTA BESSONA",EMMER +339,"RAVE PICANT, RAVE RUSTICÀ",HORSERADISH +340,RUIBARBRE,RHUBARB +341,SULLA O ENCLOVA,SULLA (FRENCH HONEYSUCKLE) +342,TARONGER AGRE,BITTER ORANGE TREE +343,VEÇA I RAIGRÀS,VETCH AND RYEGRASS +344,VIMETERA,OSIER WILLOW +345,VIVER - PRODUCTOR MVR,NURSERY (MVR PRODUCER) +346,XIRIMOIER,CHERIMOYA TREE From c4466ce61d94437b14c92563183f350aa7bed454 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 23:15:46 +0200 Subject: [PATCH 11/13] ES-MD: find RECINTO.shp wherever the archive puts it Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + fiboa_cli/datasets/es_md.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c89da358..ff0aed13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import +- ES-MD: the archive no longer nests RECINTO.shp in a folder - SK: KODKD is the (non-unique, sometimes empty) LPIS block code, keep it as block_id and use the row index as id - ES-CAT: the 2024 download is a shapefile package, not a GeoPackage; 34 crop names new in 2024 added to the mapping - CZ: find the shapefile in nested archive folders (2026) diff --git a/fiboa_cli/datasets/es_md.py b/fiboa_cli/datasets/es_md.py index a69fd6f2..4863e29a 100644 --- a/fiboa_cli/datasets/es_md.py +++ b/fiboa_cli/datasets/es_md.py @@ -4,7 +4,7 @@ class ESCLConverter(ESBaseConverter): sources = { "https://idem.comunidad.madrid/recursos_cat_geo/Catalogo/recursos/UsoDelSuelo/spacm_sigpac.cm.zip": [ - "2024_SIGPAC_shape_toda_la_com/RECINTO.shp" + "**/RECINTO.shp" ] } id = "es_md" From a43baf50fa524f0524cac02ee10a442fecd9bb33 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 23:29:16 +0200 Subject: [PATCH 12/13] EE: explicit cache names for the WFS responses Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + fiboa_cli/datasets/ee.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff0aed13..4d446321 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import +- EE: name the cached WFS responses (ee_gsaa_.gml) - ES-MD: the archive no longer nests RECINTO.shp in a folder - SK: KODKD is the (non-unique, sometimes empty) LPIS block code, keep it as block_id and use the row index as id - ES-CAT: the 2024 download is a shapefile package, not a GeoPackage; 34 crop names new in 2024 added to the mapping diff --git a/fiboa_cli/datasets/ee.py b/fiboa_cli/datasets/ee.py index 777e827b..1ec6d66b 100644 --- a/fiboa_cli/datasets/ee.py +++ b/fiboa_cli/datasets/ee.py @@ -14,10 +14,11 @@ class Convert(AddHCATMixin, FiboaBaseConverter): + # explicit cache names: the WFS URL has no usable file name variants = { - str( - year - ): f"https://kls.pria.ee/geoserver/inspire_gsaa/wfs?service=WFS&version=2.0.0&request=GetFeature&typeName=inspire_gsaa:LU.GSAA.AGRICULTURAL_PARCELS_{year}&propertyName={ATTRIBUTES}" + str(year): { + f"https://kls.pria.ee/geoserver/inspire_gsaa/wfs?service=WFS&version=2.0.0&request=GetFeature&typeName=inspire_gsaa:LU.GSAA.AGRICULTURAL_PARCELS_{year}&propertyName={ATTRIBUTES}": f"ee_gsaa_{year}.gml" + } for year in range(2024, 2009, -1) } ec_mapping_csv = "https://fiboa.org/code/ee/ee.csv" From d5287db0c01bdd28646859e25e665754986d7e83 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Sat, 22 Aug 2026 23:30:48 +0200 Subject: [PATCH 13/13] Guard the crop:code row drop (>1% is an error); Europe-LAND: crop_name as code when crop_code is empty Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 3 ++- fiboa_cli/conversion/fiboa_converter.py | 8 ++++++++ fiboa_cli/datasets/commons/euro_land.py | 8 ++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d446321..aa0d4850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - DE-BB: read the shapefile as cp1252 (its .cpg wrongly says UTF-8) - NL: new PDOK download location (rvo/gewaspercelen/atom), add the 2026 concept edition - DE-TH: note the INSPIRE download service -- Drop rows without a crop:code (required by the crop extension) with a warning instead of failing the conversion (BE-VLG 2023, ES-CN had one such row each) +- Drop rows without a crop:code (required by the crop extension) with a warning instead of failing the conversion (BE-VLG 2023, ES-CN had one such row each); more than 1% missing is an error +- Europe-LAND converters: use crop_name as crop:code when the file's crop_code column is empty (LT 2024) - BE-VLG: derive determination:datetime from the variant year instead of a constant date - `fiboa publish` no longer uploads to S3 or generates README/LICENSE files. It creates GeoParquet, PMTiles and a STAC Collection with relative links, `file:size`/`file:checksum` and a web-map-links v1.3.0 `pmtiles` link. Publishing is done by catalogs such as the [harmonized field data catalog](https://github.com/fieldsoftheworld/harmonized-field-data-catalog). - Add Italy Tuscany (IT-1) basd on EuroCrops v2 diff --git a/fiboa_cli/conversion/fiboa_converter.py b/fiboa_cli/conversion/fiboa_converter.py index 0ff5cfd5..43e3870d 100644 --- a/fiboa_cli/conversion/fiboa_converter.py +++ b/fiboa_cli/conversion/fiboa_converter.py @@ -13,6 +13,8 @@ class FiboaBaseConverter(BaseConverter): area_is_in_ha = True area_calculate_missing = False use_variant_as_determination = False + # rows lacking a REQUIRED_NON_NULL value are dropped up to this share, else it's an error + max_dropped_share = 0.01 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -32,6 +34,12 @@ def post_migrate(self, gdf): if key in targets and src in gdf.columns: nulls = gdf[src].isna() if nulls.any(): + share = nulls.mean() + if share > self.max_dropped_share: + raise ValueError( + f"{int(nulls.sum())} of {len(gdf)} rows ({share:.1%}) have no " + f"{key} ({src}); fix the converter instead of dropping them" + ) self.warning( f"Dropping {int(nulls.sum())} rows without a value for {key} ({src})" ) diff --git a/fiboa_cli/datasets/commons/euro_land.py b/fiboa_cli/datasets/commons/euro_land.py index f33bee51..5294c22e 100644 --- a/fiboa_cli/datasets/commons/euro_land.py +++ b/fiboa_cli/datasets/commons/euro_land.py @@ -47,3 +47,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) provider = "Europe-LAND HE Project " self.provider = (f"{self.provider}, {provider}") if self.provider else provider + + def migrate(self, gdf): + # Some Europe-LAND files (e.g. LT 2024) ship an empty crop_code column next to + # a populated crop_name; the name is then the best available crop code. + if "crop_code" in gdf.columns and gdf["crop_code"].isna().all(): + self.warning("crop_code is empty, using crop_name as crop:code") + gdf["crop_code"] = gdf["crop_name"] + return super().migrate(gdf)