diff --git a/csvw-eo-library/src/csvw_eo/csvw_to_smartnoise_sql.py b/csvw-eo-library/src/csvw_eo/csvw_to_smartnoise_sql.py index 6eb0760..643fcc9 100644 --- a/csvw-eo-library/src/csvw_eo/csvw_to_smartnoise_sql.py +++ b/csvw-eo-library/src/csvw_eo/csvw_to_smartnoise_sql.py @@ -192,20 +192,28 @@ def main() -> None: ---------------------- --input : str (required) Path to input CSVW-EO JSON metadata file. + --output : str (required) Path to output SmartNoise YAML metadata file. + --schema : str (default="MySchema") SmartNoise schema name. + --table : str (default="MyTable") SmartNoise table name. + --sample_max_ids : bool (default=True) Skip reservoir sampling if users appear at most max_ids times. + --censor_dims : bool (default=True) Drop GROUP BY output rows that might reveal rare individuals. + --clamp_counts : bool (default=False) Clamp negative DP counts to zero. + --clamp_columns : bool (default=True) Clamp all input data to the column lower/upper bounds. + --use_dpsu : bool (default=False) Use Differential Private Set Union for rare dimensions. """ diff --git a/csvw-eo-library/src/csvw_eo/datatypes.py b/csvw-eo-library/src/csvw_eo/datatypes.py index edc79f7..671e74e 100644 --- a/csvw-eo-library/src/csvw_eo/datatypes.py +++ b/csvw-eo-library/src/csvw_eo/datatypes.py @@ -121,7 +121,21 @@ def is_datetime(value: str) -> bool: def refine_integer_type(series: pd.Series) -> DataTypes: - """Infer type of integer.""" + """ + Refine an integer series into a more specific XML Schema datatype. + + Parameters + ---------- + series : pd.Series + Input integer series. + + Returns + ------- + DataTypes + The most specific integer subtype (e.g., positiveInteger, + negativeInteger, or integer). + + """ s = series.dropna() if (s > 0).all(): @@ -134,7 +148,25 @@ def refine_integer_type(series: pd.Series) -> DataTypes: def is_categorical(series: pd.Series, max_unique: int = 20) -> bool: - """Infer is the series is categorical (by type or number of unique values).""" + """ + Determine whether a series should be treated as categorical. + + A series is considered categorical if it is of a categorical-like + type (string/boolean) or has a small number of unique values. + + Parameters + ---------- + series : pd.Series + Input column. + max_unique : int, default=20 + Maximum number of unique values allowed for categorical inference. + + Returns + ------- + bool + True if the series is categorical, False otherwise. + + """ non_null = series.dropna() if non_null.empty: return True @@ -174,7 +206,23 @@ def is_continuous(series: pd.Series, max_unique: int = 20) -> bool: def infer_xmlschema_datatype( # noqa: PLR0911, PLR0912 series: pd.Series, ) -> DataTypes: - """Infer xml schema datatype.""" + """ + Infer the most appropriate XML Schema datatype for a pandas series. + + The inference considers pandas dtypes, string parsing, and fallback + heuristics for object types. + + Parameters + ---------- + series : pd.Series + Input column to analyze. + + Returns + ------- + DataTypes + Inferred XML Schema datatype. + + """ s = series.dropna() if s.empty: @@ -226,7 +274,25 @@ def infer_xmlschema_datatype( # noqa: PLR0911, PLR0912 def to_pandas_dtype(csvw_type: DataTypes) -> str: - """Xml datatype to pandas datatype.""" + """ + Convert a CSVW XML Schema datatype to a pandas dtype. + + Parameters + ---------- + csvw_type : DataTypes + XML Schema datatype. + + Returns + ------- + str + Equivalent pandas dtype string. + + Raises + ------ + ValueError + If the datatype is missing or invalid. + + """ if not csvw_type: raise ValueError("Missing DataTypes") @@ -251,7 +317,25 @@ def to_pandas_dtype(csvw_type: DataTypes) -> str: def to_snsql_datatype(csvw_type: DataTypes) -> str: - """Smartnoise-sql datatype to pandas datatype.""" + """ + Convert a CSVW XML Schema datatype to a SmartNoise SQL datatype. + + Parameters + ---------- + csvw_type : DataTypes + XML Schema datatype. + + Returns + ------- + str + Equivalent SmartNoise SQL datatype. + + Raises + ------ + ValueError + If the datatype is missing or invalid. + + """ if not csvw_type: raise ValueError("Missing DataTypes") diff --git a/csvw-eo-library/src/csvw_eo/generate_series.py b/csvw-eo-library/src/csvw_eo/generate_series.py index 6a37979..cbaf5db 100644 --- a/csvw-eo-library/src/csvw_eo/generate_series.py +++ b/csvw-eo-library/src/csvw_eo/generate_series.py @@ -50,7 +50,25 @@ def get_bounds(col_meta: dict[str, Any]) -> tuple[T, T]: - """Get min and max.""" + """ + Get the lower and upper bounds from column metadata. + + Parameters + ---------- + col_meta : dict[str, Any] + Column metadata containing minimum and maximum values. + + Returns + ------- + tuple[T, T] + Tuple containing the minimum and maximum bounds. + + Raises + ------ + KeyError + If the minimum or maximum bound is missing from the metadata. + + """ if MINIMUM not in col_meta: raise KeyError(f"Missing {MINIMUM} in column {col_meta[COL_NAME]}") @@ -61,14 +79,50 @@ def get_bounds(col_meta: dict[str, Any]) -> tuple[T, T]: def generate_datetime_column(col_meta: dict[str, Any], nb_rows: int, rng: np.random.Generator) -> pd.Series: - """Generate datetime column between min and max values.""" + """ + Generate a datetime column between minimum and maximum values. + + Parameters + ---------- + col_meta : dict[str, Any] + Column metadata containing datetime bounds. + nb_rows : int + Number of rows to generate. + rng : np.random.Generator + NumPy random number generator. + + Returns + ------- + pd.Series + Series containing randomly generated datetime values. + + """ lower, upper = get_bounds(col_meta) dates = pd.date_range(start=lower, end=upper) return pd.Series(rng.choice(dates, size=nb_rows)) def generate_duration_column(col_meta: dict[str, Any], nb_rows: int, rng: np.random.Generator) -> pd.Series: - """Generate duration column between min and max values.""" + """ + Generate a duration column between minimum and maximum values. + + Bounds are interpreted as seconds. + + Parameters + ---------- + col_meta : dict[str, Any] + Column metadata containing duration bounds. + nb_rows : int + Number of rows to generate. + rng : np.random.Generator + NumPy random number generator. + + Returns + ------- + pd.Series + Series containing randomly generated durations. + + """ lower, upper = get_bounds(col_meta) # assume bounds in seconds (simplest robust approach) @@ -78,7 +132,31 @@ def generate_duration_column(col_meta: dict[str, Any], nb_rows: int, rng: np.ran def generate_integer_column(col_meta: dict[str, Any], nb_rows: int, rng: np.random.Generator) -> pd.Series: - """Generate numeric column integer between min and max values respecting XSD subtype.""" + """ + Generate an integer column between minimum and maximum values. + + The generated values respect the XSD integer subtype constraints. + + Parameters + ---------- + col_meta : dict[str, Any] + Column metadata containing integer bounds and datatype. + nb_rows : int + Number of rows to generate. + rng : np.random.Generator + NumPy random number generator. + + Returns + ------- + pd.Series + Series containing randomly generated integer values. + + Notes + ----- + If zero is allowed by the bounds, at least one generated value + is forced to be zero. + + """ lower, upper = get_bounds(col_meta) datatype: DataTypes = col_meta[DATATYPE] @@ -101,18 +179,71 @@ def generate_integer_column(col_meta: dict[str, Any], nb_rows: int, rng: np.rand def generate_double_column(col_meta: dict[str, Any], nb_rows: int, rng: np.random.Generator) -> pd.Series: - """Generate numeric column double between min and max values.""" + """ + Generate a floating-point column between minimum and maximum values. + + Parameters + ---------- + col_meta : dict[str, Any] + Column metadata containing numeric bounds. + nb_rows : int + Number of rows to generate. + rng : np.random.Generator + NumPy random number generator. + + Returns + ------- + pd.Series + Series containing randomly generated floating-point values. + + """ lower, upper = get_bounds(col_meta) return pd.Series(rng.uniform(float(lower), float(upper), size=nb_rows)) def generate_boolean_column(nb_rows: int, rng: np.random.Generator) -> pd.Series: - """Generate boolean column.""" + """ + Generate a boolean column. + + Parameters + ---------- + nb_rows : int + Number of rows to generate. + rng : np.random.Generator + NumPy random number generator. + + Returns + ------- + pd.Series + Series containing randomly generated boolean values. + + """ return pd.Series(rng.choice([True, False], size=nb_rows), dtype="boolean") def generate_string_column(col_meta: dict[str, Any], nb_rows: int, rng: np.random.Generator) -> pd.Series: - """Generate string column depending on available information.""" + """ + Generate a string column based on partition metadata. + + The generated values are selected from public keys, public + partitions, or randomly generated strings depending on the + available metadata. + + Parameters + ---------- + col_meta : dict[str, Any] + Column metadata describing available partitions or keys. + nb_rows : int + Number of rows to generate. + rng : np.random.Generator + NumPy random number generator. + + Returns + ------- + pd.Series + Series containing randomly generated string values. + + """ public_keys_values = [] if KEY_VALUES in col_meta: public_keys_values = col_meta[KEY_VALUES] @@ -143,9 +274,30 @@ def generate_column_series( rng: np.random.Generator, ) -> pd.Series: """ - Generate a single column series based on metadata. + Generate a column series based on metadata and datatype. + + Supports datetime, integer, floating-point, boolean, + string, and duration datatypes. + + Parameters + ---------- + col_meta : dict[str, Any] + Column metadata describing datatype and constraints. + nb_rows : int + Number of rows to generate. + rng : np.random.Generator + NumPy random number generator. + + Returns + ------- + pd.Series + Generated pandas Series with the appropriate datatype. + + Raises + ------ + ValueError + If the datatype is unknown or unsupported. - Handles datetime, numeric, and partitioned columns, applying nulls. """ datatype: DataTypes = col_meta[DATATYPE] group = XSD_GROUP_MAP.get(datatype) @@ -342,7 +494,34 @@ def generate_dataframe( nb_rows: int, rng: np.random.Generator, ) -> pd.DataFrame: - """Generate dataframe.""" + """ + Generate a dummy dataframe based on column metadata and dependency rules. + + Columns are generated in a specified order, optionally using dependency + relationships between columns (e.g., mapping, fixed, or relational constraints). + + Parameters + ---------- + depends_map : dict[str, list[dict[str, Any]]] + Mapping of column names to their dependency definitions. + Each dependency may define how a column depends on another column. + order : list[str] + Ordered list of column names defining generation sequence. + meta_map : dict[str, dict[str, Any]] + Metadata for each column describing datatype, constraints, + and generation rules. + nb_rows : int + Number of rows to generate. + rng : np.random.Generator + NumPy random number generator used for all stochastic operations. + + Returns + ------- + pd.DataFrame + Generated dataframe with dummy random values respecting metadata + and dependency constraints. + + """ data = {} for col in order: diff --git a/csvw-eo-library/src/csvw_eo/make_dummy_from_metadata.py b/csvw-eo-library/src/csvw_eo/make_dummy_from_metadata.py index d3ddd2e..4278713 100644 --- a/csvw-eo-library/src/csvw_eo/make_dummy_from_metadata.py +++ b/csvw-eo-library/src/csvw_eo/make_dummy_from_metadata.py @@ -118,7 +118,25 @@ def column_group_partitions( df: pd.DataFrame, columns_group_meta: list[dict[str, Any]], ) -> pd.DataFrame: - """Keep only rows belonging to allowed column-group partitions.""" + """ + Filter a dataframe to keep only rows belonging to allowed column-group partitions. + + The function builds a global boolean mask by combining per-column-group + partition or key constraints, depending on metadata configuration. + + Parameters + ---------- + df : pd.DataFrame + Input dataframe to filter. + columns_group_meta : list[dict[str, Any]] + Metadata describing column groups and their partitioning rules. + + Returns + ------- + pd.DataFrame + Filtered dataframe containing only rows that satisfy all group constraints. + + """ global_mask = pd.Series(True, index=df.index) for col_group in columns_group_meta: @@ -147,7 +165,28 @@ def column_group_partitions( def apply_nulls_dataframe( df: pd.DataFrame, columns_meta: list[dict[str, Any]], rng: np.random.Generator ) -> pd.DataFrame: - """Apply null proportion on dataframe.""" + """ + Apply missing values (nulls) to a dataframe according to metadata. + + Each column is assigned null values based on its configured null + proportion and datatype-specific null handling strategy. + + Parameters + ---------- + df : pd.DataFrame + Input dataframe to modify. + columns_meta : list[dict[str, Any]] + Metadata describing each column, including null proportions + and datatype information. + rng : np.random.Generator + NumPy random number generator used for stochastic null injection. + + Returns + ------- + pd.DataFrame + Dataframe with null values applied according to metadata rules. + + """ columns_meta_map = {c[COL_NAME]: c for c in columns_meta} for col in df.columns: series = df[col] diff --git a/csvw-eo-library/src/csvw_eo/make_metadata_from_data.py b/csvw-eo-library/src/csvw_eo/make_metadata_from_data.py index 70d2be2..0a26439 100644 --- a/csvw-eo-library/src/csvw_eo/make_metadata_from_data.py +++ b/csvw-eo-library/src/csvw_eo/make_metadata_from_data.py @@ -191,7 +191,24 @@ def make_predicate(spec: dict[str, Any], value: Any) -> Predicate: # noqa: ANN4 def make_categorical_partitions( df: pd.DataFrame, privacy_unit: str, column_name: str ) -> list[SingleColumnPartition]: - """Generate partitions for a categorical column.""" + """ + Generate partitions for a categorical column. + + Parameters + ---------- + df : pd.DataFrame + Input dataframe containing the data to partition. + privacy_unit : str + Name of the privacy unit column. + column_name : str + Name of the categorical column. + + Returns + ------- + list[SingleColumnPartition] + List of generated partitions for the categorical column. + + """ partitions_meta = build_partitions( df, privacy_unit, @@ -206,7 +223,26 @@ def make_numeric_partitions( column_name: str, bounds: list[Any], ) -> list[SingleColumnPartition]: - """Generate partitions for a numeric column using provided bins.""" + """ + Generate partitions for a numeric column using predefined bounds. + + Parameters + ---------- + df : pd.DataFrame + Input dataframe containing the data to partition. + privacy_unit : str + Name of the privacy unit column. + column_name : str + Name of the numeric column. + bounds : list[Any] + List of partition boundaries used to define bins. + + Returns + ------- + list[SingleColumnPartition] + List of generated partitions for the numeric column. + + """ partitions_meta = build_partitions( df, privacy_unit, @@ -228,7 +264,29 @@ def get_multi_group_partitions( continuous_partitions: dict[str, list[Any]], privacy_unit: str, ) -> list[MultiColumnPartition]: - """Generate partitions when grouping by multiple columns.""" + """ + Generate multi-column partitions for a group of columns. + + Columns are classified as either continuous or categorical + depending on the provided partition configuration. + + Parameters + ---------- + df : pd.DataFrame + Input dataframe containing the data to partition. + col_group : list[str] + List of column names used for grouping. + continuous_partitions : dict[str, list[Any]] + Mapping of continuous column names to their partition bins. + privacy_unit : str + Name of the privacy unit column. + + Returns + ------- + list[MultiColumnPartition] + List of generated multi-column partitions. + + """ specs = [] for col in col_group: if col in continuous_partitions: @@ -371,7 +429,23 @@ def get_column_level_contribution( def build_base_column_group_kwargs( col_group: list[str], partitions_meta: list[MultiColumnPartition] ) -> dict[str, Any]: - """Return default arguments included in all column groups.""" + """ + Build the default keyword arguments for a column group. + + Parameters + ---------- + col_group : list[str] + List of column names belonging to the column group. + partitions_meta : list[MultiColumnPartition] + Metadata describing the available multi-column partitions. + + Returns + ------- + dict[str, Any] + Dictionary containing the default configuration for the + column group, including partition information and invariants. + + """ return { "columns": col_group, "public_keys_values": full_partition_to_key_multi(partitions_meta),