Skip to content

Ignored to invalid #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Overmeen wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Ignored to invalid #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 83 additions & 14 deletions hydamo_validation/logical_validation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

# %%
import numpy as np
import pandas as pd
from shapely.geometry import LineString, Point, Polygon

from hydamo_validation import general_functions, logic_functions, topologic_functions
Expand Down Expand Up @@ -95,20 +96,42 @@ def gdf_add_summary(
tags_indices,
separator=LIST_SEPARATOR,
):
gdf.loc[gdf[variable] == False, "rating"] -= penalty
gdf.loc[gdf[variable] == False, "summary"] += f"{error_message}{separator}"
gdf.loc[gdf[variable] == False, "invalid"] += f"{rule_id}{separator}"
gdf.loc[gdf[variable] == True, "valid"] += f"{rule_id}{separator}"
gdf.loc[gdf[variable].isna(), "ignored"] += f"{rule_id}{separator}"
if critical:
gdf.loc[gdf[variable] == False, "invalid_critical"] += f"{rule_id}{separator}"
else:
gdf.loc[gdf[variable] == False, "invalid_non_critical"] += (
f"{rule_id}{separator}"
)
# Cache boolean masks for efficiency
false_mask = gdf[variable] == False
true_mask = gdf[variable] == True
ignored_mask = gdf[variable].isna()

rule_str = f"{rule_id}{separator}"

# Handle False values (validation failures)
if false_mask.any():
gdf.loc[false_mask, "rating"] -= penalty
gdf.loc[false_mask, "summary"] += f"{error_message}{separator}"
gdf.loc[false_mask, "invalid"] += rule_str
if critical:
gdf.loc[false_mask, "invalid_critical"] += rule_str
else:
gdf.loc[false_mask, "invalid_non_critical"] += rule_str

# Handle True values (validation successes)
if true_mask.any():
gdf.loc[true_mask, "valid"] += rule_str

# Handle ignored (missing data) - add to invalid_critical, NOT to summary (grouped later)
if ignored_mask.any():
gdf.loc[ignored_mask, "invalid"] += rule_str
gdf.loc[ignored_mask, "invalid_critical"] += rule_str
gdf.loc[ignored_mask, "rating"] -= 5
gdf.loc[ignored_mask, "ignored"] += rule_str

# Handle tags
if tags is not None:
gdf.loc[tags_indices, ("tags_assigned")] += f"{tags}{separator}"
gdf.loc[gdf[variable] == False, "tags_invalid"] += f"{tags}{separator}"
gdf.loc[tags_indices, "tags_assigned"] += f"{tags}{separator}"
if false_mask.any():
gdf.loc[false_mask, "tags_invalid"] += f"{tags}{separator}"
if ignored_mask.any():
gdf.loc[ignored_mask, "tags_invalid"] += f"{tags}{separator}"

return gdf


Expand Down Expand Up @@ -143,6 +166,18 @@ def execute(
for col in SUMMARY_COLUMNS:
object_gdf[col] = ""

# Initialize summary with syntax issues
if "syntax_summary" in object_gdf.columns:
has_syntax = object_gdf["syntax_summary"].notna() & (
object_gdf["syntax_summary"] != ""
)
object_gdf.loc[has_syntax, "summary"] = (
object_gdf.loc[has_syntax, "syntax_summary"] + "; "
)

# Track missing columns across all rules (per feature)
object_gdf["_missing_cols"] = [set() for _ in range(len(object_gdf))]

# general rule section
if "general_rules" in object_rules.keys():
general_rules = object_rules["general_rules"]
Expand Down Expand Up @@ -251,6 +286,30 @@ def execute(

# remove all nan indices
notna_indices = _notna_indices(object_gdf, input_variables)
missing_data_indices = [
idx for idx in indices if idx not in notna_indices
]

# Accumulate missing columns per feature (identify which columns each feature is missing)
if missing_data_indices:
# Collect all columns to check from input_variables
cols_to_check = []
for k, v in input_variables.items():
if k not in NOTNA_COL_IGNORE:
cols_to_check.extend([v] if not isinstance(v, list) else v)
cols_to_check = [
c for c in cols_to_check if c in object_gdf.columns
]

# For each feature, identify which specific columns are missing
for idx in missing_data_indices:
missing_for_idx = [
col
for col in cols_to_check
if pd.isna(object_gdf.at[idx, col])
]
object_gdf.at[idx, "_missing_cols"].update(missing_for_idx)

indices = [i for i in indices[indices.notna()] if i in notna_indices]

# add object_relation
Expand Down Expand Up @@ -346,6 +405,14 @@ def execute(
else:
pass

# Add grouped missing data message once per feature
has_missing = object_gdf["_missing_cols"].apply(len) > 0
for idx in object_gdf[has_missing].index:
cols_list = sorted(object_gdf.at[idx, "_missing_cols"])
cols_str = ", ".join(cols_list)
msg = f"ontbrekende waarde{'n' if len(cols_list) > 1 else ''}: {cols_str}"
object_gdf.loc[idx, "summary"] += f"{msg}{LIST_SEPARATOR}"

# drop columns
drop_columns = [
i
Expand All @@ -355,7 +422,9 @@ def execute(
+ ["nen3610id", "geometry", "rating"]
+ SUMMARY_COLUMNS
]
object_gdf.drop(columns=drop_columns, inplace=True)
# Also drop temporary tracking columns
drop_columns.extend(["_missing_cols", "syntax_summary"])
object_gdf.drop(columns=drop_columns, inplace=True, errors="ignore")
# re_order columns
column_order = ["nen3610id"]
column_order += list(col_translation.keys())
Expand Down
36 changes: 33 additions & 3 deletions hydamo_validation/validator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,8 @@ def _validator(
if not path.exists():
missing_paths += [str(path)]
if missing_paths:
result_summary.error += [f'missing_paths: {",".join(missing_paths)}']
raise FileNotFoundError(f'missing_paths: {",".join(missing_paths)}')
result_summary.error += [f"missing_paths: {','.join(missing_paths)}"]
raise FileNotFoundError(f"missing_paths: {','.join(missing_paths)}")
else:
validation_rules_sets = read_validation_rules(
validation_rules_json, result_summary
Expand All @@ -232,7 +232,7 @@ def _validator(
]
if unsupported_output_types:
error_message = (
r"unsupported output types: " f'{",".join(unsupported_output_types)}'
r"unsupported output types: " f"{','.join(unsupported_output_types)}"
)
result_summary.error += [error_message]
raise TypeError(error_message)
Expand Down Expand Up @@ -301,6 +301,36 @@ def _validator(
keep_columns=INCLUDE_COLUMNS,
)

# Initialize syntax_summary column
result_gdf["syntax_summary"] = ""

# Create readable syntax summary for critical errors
if "syntax_geometry" in result_gdf.columns:
has_geom_error = result_gdf["syntax_geometry"].str.contains(
"7", na=False
)
result_gdf.loc[has_geom_error, "syntax_summary"] = (
"geometrie is invalid"
)

# Check for unconvertable errors (fout 1)
error_messages = []
for col in result_gdf.filter(regex="^syntax_").columns:
if col not in ["syntax_geometry", "syntax_oordeel", "syntax_summary"]:
has_error_1 = result_gdf[col].str.contains("1", na=False)
if has_error_1.any():
col_name = col.replace("syntax_", "")
error_msg = f"{col_name} niet converteerbaar"
# Build messages per row more efficiently
for idx in result_gdf[has_error_1].index:
current = result_gdf.at[idx, "syntax_summary"]
result_gdf.at[idx, "syntax_summary"] = (
f"{current}; {error_msg}" if current else error_msg
)

# Copy to gdf
gdf["syntax_summary"] = result_gdf["syntax_summary"]

# Add the syntax-validation result to the results_summary
layers_summary.set_data(result_gdf, layer, schema["geometry"])
# Add the corrected datasets_layer data to the datamodel.
Expand Down