Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions csvw-eo-library/src/csvw_eo/csvw_to_smartnoise_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,20 +192,28 @@ def main() -> None:
----------------------
--input : str (required)
Path to input CSVW-EO JSON metadata file.

--output : str (required)
Path to output SmartNoise YAML metadata file.

--schema : str (default="MySchema")
SmartNoise schema name.

--table : str (default="MyTable")
SmartNoise table name.

--sample_max_ids : bool (default=True)
Skip reservoir sampling if users appear at most max_ids times.

--censor_dims : bool (default=True)
Drop GROUP BY output rows that might reveal rare individuals.

--clamp_counts : bool (default=False)
Clamp negative DP counts to zero.

--clamp_columns : bool (default=True)
Clamp all input data to the column lower/upper bounds.

--use_dpsu : bool (default=False)
Use Differential Private Set Union for rare dimensions.
"""
Expand Down
94 changes: 89 additions & 5 deletions csvw-eo-library/src/csvw_eo/datatypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,21 @@ def is_datetime(value: str) -> bool:


def refine_integer_type(series: pd.Series) -> DataTypes:
"""Infer type of integer."""
"""
Refine an integer series into a more specific XML Schema datatype.

Parameters
----------
series : pd.Series
Input integer series.

Returns
-------
DataTypes
The most specific integer subtype (e.g., positiveInteger,
negativeInteger, or integer).

"""
s = series.dropna()

if (s > 0).all():
Expand All @@ -134,7 +148,25 @@ def refine_integer_type(series: pd.Series) -> DataTypes:


def is_categorical(series: pd.Series, max_unique: int = 20) -> bool:
"""Infer is the series is categorical (by type or number of unique values)."""
"""
Determine whether a series should be treated as categorical.

A series is considered categorical if it is of a categorical-like
type (string/boolean) or has a small number of unique values.

Parameters
----------
series : pd.Series
Input column.
max_unique : int, default=20
Maximum number of unique values allowed for categorical inference.

Returns
-------
bool
True if the series is categorical, False otherwise.

"""
non_null = series.dropna()
if non_null.empty:
return True
Expand Down Expand Up @@ -174,7 +206,23 @@ def is_continuous(series: pd.Series, max_unique: int = 20) -> bool:
def infer_xmlschema_datatype( # noqa: PLR0911, PLR0912
series: pd.Series,
) -> DataTypes:
"""Infer xml schema datatype."""
"""
Infer the most appropriate XML Schema datatype for a pandas series.

The inference considers pandas dtypes, string parsing, and fallback
heuristics for object types.

Parameters
----------
series : pd.Series
Input column to analyze.

Returns
-------
DataTypes
Inferred XML Schema datatype.

"""
s = series.dropna()

if s.empty:
Expand Down Expand Up @@ -226,7 +274,25 @@ def infer_xmlschema_datatype( # noqa: PLR0911, PLR0912


def to_pandas_dtype(csvw_type: DataTypes) -> str:
"""Xml datatype to pandas datatype."""
"""
Convert a CSVW XML Schema datatype to a pandas dtype.

Parameters
----------
csvw_type : DataTypes
XML Schema datatype.

Returns
-------
str
Equivalent pandas dtype string.

Raises
------
ValueError
If the datatype is missing or invalid.

"""
if not csvw_type:
raise ValueError("Missing DataTypes")

Expand All @@ -251,7 +317,25 @@ def to_pandas_dtype(csvw_type: DataTypes) -> str:


def to_snsql_datatype(csvw_type: DataTypes) -> str:
"""Smartnoise-sql datatype to pandas datatype."""
"""
Convert a CSVW XML Schema datatype to a SmartNoise SQL datatype.

Parameters
----------
csvw_type : DataTypes
XML Schema datatype.

Returns
-------
str
Equivalent SmartNoise SQL datatype.

Raises
------
ValueError
If the datatype is missing or invalid.

"""
if not csvw_type:
raise ValueError("Missing DataTypes")

Expand Down
Loading
Loading