From e970a0bde4e5c14e3a3f8a3c7531d900e923e90a Mon Sep 17 00:00:00 2001 From: Zohair Shafi Date: Thu, 24 Jul 2025 11:17:18 -0400 Subject: [PATCH 01/10] Create kl_divergence.py Signed-off-by: Zohair Shafi --- feature/kl_divergence.py | 69 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 feature/kl_divergence.py diff --git a/feature/kl_divergence.py b/feature/kl_divergence.py new file mode 100644 index 0000000..ca7cea5 --- /dev/null +++ b/feature/kl_divergence.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# Copyright FMR LLC +# SPDX-License-Identifier: Apache-2.0 + +from typing import NoReturn, Tuple + +import pandas as pd +import numpy as np + +from scipy.special import kl_div, rel_entr +from feature.base import _BaseSupervisedSelector, _BaseDispatcher +from feature.utils import Num, get_task_string, check_true + + +from tqdm import tqdm + + +class _KL_Divergence(_BaseSupervisedSelector, _BaseDispatcher): + + def __init__(self, seed: int, num_features: Num,): + super().__init__(seed) + + self.num_features = num_features # this could be int or float + + # Implementor is decided when data becomes available in fit() + self.imp = None + + def get_model_args(self, selection_method) -> Tuple: + + # Pack model argument + return selection_method.num_features + + def dispatch_model(self, labels: pd.Series, *args): + + # Unpack model argument + num_features = args[0] + self.num_features = num_features + + + def fit(self, data: pd.DataFrame, labels: pd.Series) -> NoReturn: + + check_true(len(np.unique(labels)) == 2, TypeError("Only binary labels are supported for KL Divergence")) + + kl_mat = np.zeros((data.shape[1], 1)) + data = data.values + label_categories = np.unique(labels) + + for i in tqdm(range(data.shape[1])): + + pos_idx = np.where(labels == label_categories[0])[0] + neg_idx = np.where(labels == label_categories[1])[0] + + f1 = np.histogram(data[pos_idx, i], bins = 100)[0] + f2 = np.histogram(data[neg_idx, i], bins = 100)[0] + + f1 = f1 / np.sum(f1) + f2 = f2 / np.sum(f2) + + kl = rel_entr(f1, f2) + kl[kl == np.inf] = 0 + + kl_mat[i] = np.sum(kl) + + self.abs_scores = kl_mat.flatten() + + def transform(self, data: pd.DataFrame) -> pd.DataFrame: + + # Select top-k from data based on abs_scores and num_features + return self.get_top_k(data, self.abs_scores) From 1fc56ae985fb43d98868395973e399b5d128bf47 Mon Sep 17 00:00:00 2001 From: Zohair Shafi Date: Thu, 24 Jul 2025 11:18:27 -0400 Subject: [PATCH 02/10] Update selector.py Signed-off-by: Zohair Shafi --- feature/selector.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/feature/selector.py b/feature/selector.py index 80cf152..46e6216 100644 --- a/feature/selector.py +++ b/feature/selector.py @@ -26,6 +26,7 @@ from feature.base import _BaseDispatcher, _BaseSupervisedSelector, _BaseUnsupervisedSelector from feature.correlation import _Correlation from feature.linear import _Linear +from feature.kl_divergence import _KL_Divergence from feature.statistical import _Statistical from feature.text_based import _TextBased from feature.tree_based import _TreeBased @@ -166,6 +167,28 @@ def _validate(self): check_true(isinstance(self.alpha, (int, float)), TypeError("Alpha must a number.")) check_true(self.alpha >= 0, ValueError("Alpha cannot be negative")) + class KL_Divergence(NamedTuple): + """ + + Computes the distribution of a given feature for instances where label == 1 and label == 0. + Uses KL divergence between the two distributions as an importance score, + where a higher value indicates greater discriminative power of the feature + with respect to the binary label. + + Attributes + ---------- + num_features: Num, optional + If integer, select top num_features. + If float, select the top num_features percentile. + """ + num_features: Num = 0.0 + + def _validate(self): + check_true(isinstance(self.num_features, (int, float)), TypeError("Num features must a number.")) + check_true(self.num_features > 0, ValueError("Num features must be greater than zero.")) + if isinstance(self.num_features, float): + check_true(self.num_features <= 1, ValueError("Num features ratio must be between [0..1].")) + class Statistical(NamedTuple): """ Supervised feature selector based on statistical tests. @@ -480,6 +503,7 @@ class Selective: def __init__(self, selection_method: Union[SelectionMethod.Correlation, SelectionMethod.Linear, + SelectionMethod.KL_Divergence, SelectionMethod.TreeBased, SelectionMethod.TextBased, SelectionMethod.Statistical, @@ -521,6 +545,9 @@ def __init__(self, selection_method: Union[SelectionMethod.Correlation, self._imp: Union[None, _BaseUnsupervisedSelector, _BaseSupervisedSelector] = None if isinstance(selection_method, SelectionMethod.Correlation): self._imp = _Correlation(self.seed, self.selection_method.threshold, self.selection_method.method) + + elif isinstance(selection_method, SelectionMethod.KL_Divergence): + self._imp = _KL_Divergence(self.seed, self.selection_method.num_features) elif isinstance(selection_method, SelectionMethod.Linear): self._imp = _Linear(self.seed, self.selection_method.num_features, self.selection_method.regularization, self.selection_method.alpha) @@ -593,6 +620,7 @@ def _validate_args(seed, selection_method) -> NoReturn: # Selection Method type check_true(isinstance(selection_method, (SelectionMethod.Correlation, SelectionMethod.Linear, + SelectionMethod.KL_Divergence, SelectionMethod.TextBased, SelectionMethod.TreeBased, SelectionMethod.Statistical, From 0a238e024b09e83f76947341b03d4019da3e2b9d Mon Sep 17 00:00:00 2001 From: zohairshafi Date: Thu, 7 Aug 2025 16:56:03 -0400 Subject: [PATCH 03/10] Updates --- CHANGELOG.txt | 6 ++++ README.md | 3 +- feature/_version.py | 2 +- feature/kl_divergence.py | 62 +++++++++++++++++----------------------- feature/selector.py | 39 ++++++------------------- feature/statistical.py | 10 ++++++- tests/test_benchmark.py | 1 + 7 files changed, 54 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 48a1b4a..2a42bb8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,12 @@ CHANGELOG ========= +------------------------------------------------------------------------------- +August 7, 2025 1.2.0 +------------------------------------------------------------------------------- + +- Added KL Divergence based feature selection for binary labels. Thanks to @zohairshafi for contributing this method. + ------------------------------------------------------------------------------- April, 24, 2023 1.1.2 ------------------------------------------------------------------------------- diff --git a/README.md b/README.md index f4b0bcb..e05dc21 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ print("Scores:", list(selector.get_absolute_scores())) |:--------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| | [Variance per Feature](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.VarianceThreshold.html) | `threshold` | | [Correlation pairwise Features](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.corr.html) | [Pearson Correlation Coefficient](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient)
[Kendall Rank Correlation Coefficient](https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient)
[Spearman's Rank Correlation Coefficient](https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient)
| -| [Statistical Analysis](https://scikit-learn.org/stable/modules/feature_selection.html#univariate-feature-selection) | [ANOVA F-test Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_classif.html)
[F-value Regression](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_regression.html)
[Chi-Square](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.chi2.html)
[Mutual Information Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.mutual_info_classif.html)
[Variance Inflation Factor](https://www.statsmodels.org/stable/generated/statsmodels.stats.outliers_influence.variance_inflation_factor.html) | +| [Statistical Analysis](https://scikit-learn.org/stable/modules/feature_selection.html#univariate-feature-selection) | [ANOVA F-test Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_classif.html)
[F-value Regression](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_regression.html)
[Chi-Square](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.chi2.html)
[Mutual Information Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.mutual_info_classif.html)
[Variance Inflation Factor](https://www.statsmodels.org/stable/generated/statsmodels.stats.outliers_influence.variance_inflation_factor.html)
[KL Divergence](https://en.wikipedia.org/wiki/Kullback–Leibler_divergence) | | [Linear Methods](https://en.wikipedia.org/wiki/Linear_regression) | [Linear Regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html?highlight=linear%20regression#sklearn.linear_model.LinearRegression)
[Logistic Regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html?highlight=logistic%20regression#sklearn.linear_model.LogisticRegression)
[Lasso Regularization](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html#sklearn.linear_model.Lasso)
[Ridge Regularization](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge)
| | [Tree-based Methods](https://scikit-learn.org/stable/modules/tree.html) | [Decision Tree](https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html#sklearn.tree.DecisionTreeClassifier)
[Random Forest](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html?highlight=random%20forest#sklearn.ensemble.RandomForestClassifier)
[Extra Trees Classifier](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html)
[XGBoost](https://xgboost.readthedocs.io/en/latest/)
[LightGBM](https://lightgbm.readthedocs.io/en/latest/)
[AdaBoost](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html)
[CatBoost](https://github.com/catboost)
[Gradient Boosting Tree](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html)
| | [Text-based Methods](https://link.springer.com/chapter/10.1007/978-3-030-78230-6_27) | `featurization_method` = [TextWiser](https://github.com/fidelity/textwiser)
`optimization_method = ["exact", "greedy", "kmeans", "random"]`
`cost_metric = ["unicost", "diverse"]` | @@ -82,6 +82,7 @@ selectors = { "stat_anova": SelectionMethod.Statistical(num_features, method="anova"), "stat_chi_square": SelectionMethod.Statistical(num_features, method="chi_square"), "stat_mutual_info": SelectionMethod.Statistical(num_features, method="mutual_info"), + "stat_kl_divergence": SelectionMethod.Statistical(num_features, method="kl_divergence"), # Linear methods "linear": SelectionMethod.Linear(num_features, regularization="none"), diff --git a/feature/_version.py b/feature/_version.py index 35e9165..0f3c50c 100644 --- a/feature/_version.py +++ b/feature/_version.py @@ -2,4 +2,4 @@ # Copyright FMR LLC # SPDX-License-Identifier: Apache-2.0 -__version__ = "1.1.2" \ No newline at end of file +__version__ = "1.2.0" \ No newline at end of file diff --git a/feature/kl_divergence.py b/feature/kl_divergence.py index ca7cea5..6afe508 100644 --- a/feature/kl_divergence.py +++ b/feature/kl_divergence.py @@ -7,61 +7,51 @@ import pandas as pd import numpy as np -from scipy.special import kl_div, rel_entr -from feature.base import _BaseSupervisedSelector, _BaseDispatcher -from feature.utils import Num, get_task_string, check_true +from scipy.special import rel_entr +from feature.base import _BaseSupervisedSelector +from feature.utils import Num, check_true -from tqdm import tqdm +class _KL_Divergence(_BaseSupervisedSelector): - -class _KL_Divergence(_BaseSupervisedSelector, _BaseDispatcher): - - def __init__(self, seed: int, num_features: Num,): + def __init__(self, seed: int, num_features: Num, num_bins: Num = 100): super().__init__(seed) self.num_features = num_features # this could be int or float + self.num_bins = num_bins - # Implementor is decided when data becomes available in fit() - self.imp = None - - def get_model_args(self, selection_method) -> Tuple: - - # Pack model argument - return selection_method.num_features - - def dispatch_model(self, labels: pd.Series, *args): + def fit(self, X: pd.DataFrame, y: pd.Series) -> NoReturn: - # Unpack model argument - num_features = args[0] - self.num_features = num_features + check_true(len(np.unique(y)) == 2, TypeError("Only binary labels are supported for KL Divergence")) + kl_mat = np.zeros((self.num_features, 1)) + X = X.values + label_categories = np.unique(y) - def fit(self, data: pd.DataFrame, labels: pd.Series) -> NoReturn: - - check_true(len(np.unique(labels)) == 2, TypeError("Only binary labels are supported for KL Divergence")) - - kl_mat = np.zeros((data.shape[1], 1)) - data = data.values - label_categories = np.unique(labels) + class_one_idx = np.where(y == label_categories[0])[0] + class_two_idx = np.where(y == label_categories[1])[0] - for i in tqdm(range(data.shape[1])): - - pos_idx = np.where(labels == label_categories[0])[0] - neg_idx = np.where(labels == label_categories[1])[0] + for i in range(self.num_features): - f1 = np.histogram(data[pos_idx, i], bins = 100)[0] - f2 = np.histogram(data[neg_idx, i], bins = 100)[0] + f1 = np.histogram(X[class_one_idx, i], bins = self.num_bins)[0] + f2 = np.histogram(X[class_two_idx, i], bins = self.num_bins)[0] f1 = f1 / np.sum(f1) f2 = f2 / np.sum(f2) + # KL Divergence is not symmetric, so we calculate divergence in both directions kl = rel_entr(f1, f2) - kl[kl == np.inf] = 0 + kl_reversed = rel_entr(f2, f1) + + kl[kl == np.inf] = 9999 + kl_reversed[kl_reversed == np.inf] = 9999 - kl_mat[i] = np.sum(kl) + kl_mat[i] = np.sum(kl) + np.sum(kl_reversed) - self.abs_scores = kl_mat.flatten() + scores_ = kl_mat.flatten() + + self.scores_ = scores_ # This is used by the statistical.py fit function. + self.abs_scores = scores_ def transform(self, data: pd.DataFrame) -> pd.DataFrame: diff --git a/feature/selector.py b/feature/selector.py index 46e6216..f137458 100644 --- a/feature/selector.py +++ b/feature/selector.py @@ -26,7 +26,6 @@ from feature.base import _BaseDispatcher, _BaseSupervisedSelector, _BaseUnsupervisedSelector from feature.correlation import _Correlation from feature.linear import _Linear -from feature.kl_divergence import _KL_Divergence from feature.statistical import _Statistical from feature.text_based import _TextBased from feature.tree_based import _TreeBased @@ -37,7 +36,7 @@ __author__ = "FMR LLC" -__version__ = "1.0.0" +__version__ = "1.2.0" __copyright__ = "Copyright (C), FMR LLC" @@ -167,28 +166,6 @@ def _validate(self): check_true(isinstance(self.alpha, (int, float)), TypeError("Alpha must a number.")) check_true(self.alpha >= 0, ValueError("Alpha cannot be negative")) - class KL_Divergence(NamedTuple): - """ - - Computes the distribution of a given feature for instances where label == 1 and label == 0. - Uses KL divergence between the two distributions as an importance score, - where a higher value indicates greater discriminative power of the feature - with respect to the binary label. - - Attributes - ---------- - num_features: Num, optional - If integer, select top num_features. - If float, select the top num_features percentile. - """ - num_features: Num = 0.0 - - def _validate(self): - check_true(isinstance(self.num_features, (int, float)), TypeError("Num features must a number.")) - check_true(self.num_features > 0, ValueError("Num features must be greater than zero.")) - if isinstance(self.num_features, float): - check_true(self.num_features <= 1, ValueError("Num features ratio must be between [0..1].")) - class Statistical(NamedTuple): """ Supervised feature selector based on statistical tests. @@ -225,6 +202,13 @@ class Statistical(NamedTuple): searching for the optimal binning strategy. Note: MIC is dropped from Selective due to inactive MINE library + The KL Divergence feature importance should only be used with + binary labels. It computes the distribution of a given feature for instances where label == 1 and label == 0. + Uses KL divergence between the two distributions as an importance score, + where a higher value indicates greater discriminative power of the feature + with respect to the binary label. Since KL Divergence is non-symmetric, this method + computer the divergence in both directions and sums them up. + Notes on Randomness: - Mutual Info is non-deterministic, depends on the seed value. - The other methods are deterministic @@ -250,7 +234,7 @@ def _validate(self): if isinstance(self.num_features, float): check_true(self.num_features <= 1, ValueError("Num features ratio must be between [0..1].")) # "maximal_info" dropped - check_true(self.method in ["anova", "chi_square", "mutual_info", "variance_inflation"], + check_true(self.method in ["anova", "chi_square", "mutual_info", "variance_inflation", "kl_divergence"], ValueError("Statistical method can only be anova, chi_square, or mutual_info.")) class TreeBased(NamedTuple): @@ -503,7 +487,6 @@ class Selective: def __init__(self, selection_method: Union[SelectionMethod.Correlation, SelectionMethod.Linear, - SelectionMethod.KL_Divergence, SelectionMethod.TreeBased, SelectionMethod.TextBased, SelectionMethod.Statistical, @@ -545,9 +528,6 @@ def __init__(self, selection_method: Union[SelectionMethod.Correlation, self._imp: Union[None, _BaseUnsupervisedSelector, _BaseSupervisedSelector] = None if isinstance(selection_method, SelectionMethod.Correlation): self._imp = _Correlation(self.seed, self.selection_method.threshold, self.selection_method.method) - - elif isinstance(selection_method, SelectionMethod.KL_Divergence): - self._imp = _KL_Divergence(self.seed, self.selection_method.num_features) elif isinstance(selection_method, SelectionMethod.Linear): self._imp = _Linear(self.seed, self.selection_method.num_features, self.selection_method.regularization, self.selection_method.alpha) @@ -620,7 +600,6 @@ def _validate_args(seed, selection_method) -> NoReturn: # Selection Method type check_true(isinstance(selection_method, (SelectionMethod.Correlation, SelectionMethod.Linear, - SelectionMethod.KL_Divergence, SelectionMethod.TextBased, SelectionMethod.TreeBased, SelectionMethod.Statistical, diff --git a/feature/statistical.py b/feature/statistical.py index fec4844..141fe6a 100644 --- a/feature/statistical.py +++ b/feature/statistical.py @@ -10,9 +10,11 @@ import pandas as pd from sklearn.feature_selection import chi2, f_classif, f_regression, mutual_info_classif, mutual_info_regression from statsmodels.stats.outliers_influence import variance_inflation_factor +from scipy.special import rel_entr from feature.base import _BaseSupervisedSelector, _BaseDispatcher from feature.utils import get_selector, Num, get_task_string +from feature.kl_divergence import _KL_Divergence class _Statistical(_BaseSupervisedSelector, _BaseDispatcher): @@ -41,7 +43,8 @@ def __init__(self, seed: int, num_features: Num, method: str): "classification_chi_square": chi2, "classification_mutual_info": partial(mutual_info_classif, random_state=self.seed), # "classification_maximal_info": MINE(), # dropped - "unsupervised_variance_inflation": variance_inflation_factor} + "unsupervised_variance_inflation": variance_inflation_factor, + "kl_divergence" : _KL_Divergence(num_features = self.num_features, seed = self.seed)} def get_model_args(self, selection_method) -> Tuple: @@ -56,6 +59,8 @@ def dispatch_model(self, labels: pd.Series, *args): # Get statistical scoring function if method == "variance_inflation": score_func = self.factory.get("unsupervised_" + method) + elif method == "kl_divergence": + score_func = self.factory.get(method) else: score_func = self.factory.get(get_task_string(labels) + method) @@ -64,6 +69,8 @@ def dispatch_model(self, labels: pd.Series, *args): raise TypeError(method + " cannot be used for task: " + get_task_string(labels)) elif method == "variance_inflation": # or isinstance(score_func, MINE) (dropped) self.imp = score_func + elif method == "kl_divergence": + self.imp = score_func else: # Set sklearn model selector based on scoring function self.imp = get_selector(score_func, self.num_features) @@ -82,6 +89,7 @@ def fit(self, data: pd.DataFrame, labels: pd.Series) -> NoReturn: if self.method == "variance_inflation": # VIF is unsupervised, regression between data and each feature self.abs_scores = np.array([variance_inflation_factor(data.values, i) for i in range(data.shape[1])]) + else: # sklearn selector model self.imp.fit(X=data, y=labels) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 149da2c..a795ea8 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -29,6 +29,7 @@ class TestBenchmark(BaseTest): "univ_anova": SelectionMethod.Statistical(num_features, method="anova"), "univ_chi_square": SelectionMethod.Statistical(num_features, method="chi_square"), "univ_mutual_info": SelectionMethod.Statistical(num_features, method="mutual_info"), + "kl_divergence": SelectionMethod.Statistical(num_features, method="kl_divergence"), "linear": SelectionMethod.Linear(num_features, regularization="none"), "lasso": SelectionMethod.Linear(num_features, regularization="lasso", alpha=alpha), "ridge": SelectionMethod.Linear(num_features, regularization="ridge", alpha=alpha), From bf09e26aa6220eaa2f595adfd5e8cf9deb9cf48c Mon Sep 17 00:00:00 2001 From: zohairshafi Date: Fri, 8 Aug 2025 13:11:36 -0400 Subject: [PATCH 04/10] Updates --- README.md | 4 +- feature/kl_divergence.py | 6 +-- feature/selector.py | 2 +- feature/statistical.py | 23 ++++++------ tests/test_stat_kl.py | 79 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 tests/test_stat_kl.py diff --git a/README.md b/README.md index e05dc21..a07ba76 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ print("Scores:", list(selector.get_absolute_scores())) |:--------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| | [Variance per Feature](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.VarianceThreshold.html) | `threshold` | | [Correlation pairwise Features](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.corr.html) | [Pearson Correlation Coefficient](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient)
[Kendall Rank Correlation Coefficient](https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient)
[Spearman's Rank Correlation Coefficient](https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient)
| -| [Statistical Analysis](https://scikit-learn.org/stable/modules/feature_selection.html#univariate-feature-selection) | [ANOVA F-test Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_classif.html)
[F-value Regression](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_regression.html)
[Chi-Square](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.chi2.html)
[Mutual Information Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.mutual_info_classif.html)
[Variance Inflation Factor](https://www.statsmodels.org/stable/generated/statsmodels.stats.outliers_influence.variance_inflation_factor.html)
[KL Divergence](https://en.wikipedia.org/wiki/Kullback–Leibler_divergence) | +| [Statistical Analysis](https://scikit-learn.org/stable/modules/feature_selection.html#univariate-feature-selection) | [ANOVA F-test Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_classif.html)
[F-value Regression](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_regression.html)
[Chi-Square](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.chi2.html)
[KL Divergence](https://en.wikipedia.org/wiki/Kullback–Leibler_divergence)
[Mutual Information Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.mutual_info_classif.html)
[Variance Inflation Factor](https://www.statsmodels.org/stable/generated/statsmodels.stats.outliers_influence.variance_inflation_factor.html) | | [Linear Methods](https://en.wikipedia.org/wiki/Linear_regression) | [Linear Regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html?highlight=linear%20regression#sklearn.linear_model.LinearRegression)
[Logistic Regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html?highlight=logistic%20regression#sklearn.linear_model.LogisticRegression)
[Lasso Regularization](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html#sklearn.linear_model.Lasso)
[Ridge Regularization](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge)
| | [Tree-based Methods](https://scikit-learn.org/stable/modules/tree.html) | [Decision Tree](https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html#sklearn.tree.DecisionTreeClassifier)
[Random Forest](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html?highlight=random%20forest#sklearn.ensemble.RandomForestClassifier)
[Extra Trees Classifier](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html)
[XGBoost](https://xgboost.readthedocs.io/en/latest/)
[LightGBM](https://lightgbm.readthedocs.io/en/latest/)
[AdaBoost](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html)
[CatBoost](https://github.com/catboost)
[Gradient Boosting Tree](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html)
| | [Text-based Methods](https://link.springer.com/chapter/10.1007/978-3-030-78230-6_27) | `featurization_method` = [TextWiser](https://github.com/fidelity/textwiser)
`optimization_method = ["exact", "greedy", "kmeans", "random"]`
`cost_metric = ["unicost", "diverse"]` | @@ -81,8 +81,8 @@ selectors = { # Statistical methods "stat_anova": SelectionMethod.Statistical(num_features, method="anova"), "stat_chi_square": SelectionMethod.Statistical(num_features, method="chi_square"), - "stat_mutual_info": SelectionMethod.Statistical(num_features, method="mutual_info"), "stat_kl_divergence": SelectionMethod.Statistical(num_features, method="kl_divergence"), + "stat_mutual_info": SelectionMethod.Statistical(num_features, method="mutual_info"), # Linear methods "linear": SelectionMethod.Linear(num_features, regularization="none"), diff --git a/feature/kl_divergence.py b/feature/kl_divergence.py index 6afe508..06c9d50 100644 --- a/feature/kl_divergence.py +++ b/feature/kl_divergence.py @@ -22,12 +22,12 @@ def __init__(self, seed: int, num_features: Num, num_bins: Num = 100): def fit(self, X: pd.DataFrame, y: pd.Series) -> NoReturn: - check_true(len(np.unique(y)) == 2, TypeError("Only binary labels are supported for KL Divergence")) + label_categories = np.unique(y) + check_true(len(label_categories) == 2, TypeError("Only binary labels are supported for KL Divergence")) kl_mat = np.zeros((self.num_features, 1)) X = X.values - label_categories = np.unique(y) - + class_one_idx = np.where(y == label_categories[0])[0] class_two_idx = np.where(y == label_categories[1])[0] diff --git a/feature/selector.py b/feature/selector.py index f137458..75f213e 100644 --- a/feature/selector.py +++ b/feature/selector.py @@ -234,7 +234,7 @@ def _validate(self): if isinstance(self.num_features, float): check_true(self.num_features <= 1, ValueError("Num features ratio must be between [0..1].")) # "maximal_info" dropped - check_true(self.method in ["anova", "chi_square", "mutual_info", "variance_inflation", "kl_divergence"], + check_true(self.method in ["anova", "chi_square", "kl_divergence", "mutual_info", "variance_inflation"], ValueError("Statistical method can only be anova, chi_square, or mutual_info.")) class TreeBased(NamedTuple): diff --git a/feature/statistical.py b/feature/statistical.py index 141fe6a..3f4288b 100644 --- a/feature/statistical.py +++ b/feature/statistical.py @@ -35,16 +35,17 @@ def __init__(self, seed: int, num_features: Num, method: str): self.imp = None # Implementor factory - self.factory = {"regression_anova": f_regression, - "regression_chi_square": None, - "regression_mutual_info": partial(mutual_info_regression, random_state=self.seed), - # "regression_maximal_info": MINE(), # dropped - "classification_anova": f_classif, + self.factory = {"classification_anova": f_classif, "classification_chi_square": chi2, "classification_mutual_info": partial(mutual_info_classif, random_state=self.seed), # "classification_maximal_info": MINE(), # dropped + "kl_divergence" : _KL_Divergence(num_features = self.num_features, seed = self.seed), + "regression_anova": f_regression, + "regression_chi_square": None, + "regression_mutual_info": partial(mutual_info_regression, random_state=self.seed), + # "regression_maximal_info": MINE(), # dropped "unsupervised_variance_inflation": variance_inflation_factor, - "kl_divergence" : _KL_Divergence(num_features = self.num_features, seed = self.seed)} + } def get_model_args(self, selection_method) -> Tuple: @@ -57,20 +58,20 @@ def dispatch_model(self, labels: pd.Series, *args): method = args[0] # Get statistical scoring function - if method == "variance_inflation": - score_func = self.factory.get("unsupervised_" + method) - elif method == "kl_divergence": + if method == "kl_divergence": score_func = self.factory.get(method) + elif method == "variance_inflation": + score_func = self.factory.get("unsupervised_" + method) else: score_func = self.factory.get(get_task_string(labels) + method) # Check scoring compatibility with task if score_func is None: raise TypeError(method + " cannot be used for task: " + get_task_string(labels)) - elif method == "variance_inflation": # or isinstance(score_func, MINE) (dropped) - self.imp = score_func elif method == "kl_divergence": self.imp = score_func + elif method == "variance_inflation": # or isinstance(score_func, MINE) (dropped) + self.imp = score_func else: # Set sklearn model selector based on scoring function self.imp = get_selector(score_func, self.num_features) diff --git a/tests/test_stat_kl.py b/tests/test_stat_kl.py new file mode 100644 index 0000000..d1aa506 --- /dev/null +++ b/tests/test_stat_kl.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +# Copyright FMR LLC +# SPDX-License-Identifier: Apache-2.0 + +from sklearn.datasets import fetch_california_housing, load_iris +from feature.utils import get_data_label +from feature.selector import Selective, SelectionMethod +from tests.test_base import BaseTest + + +class TestKL(BaseTest): + + def test_kl_regress_invalid(self): + data, label = get_data_label(fetch_california_housing()) + data = data.drop(columns=["Latitude", "Longitude", "Population"]) + + method = SelectionMethod.Statistical(num_features=3, method="kl_divergence") + selector = Selective(method) + with self.assertRaises(TypeError): + selector.fit(data, label) + + def test_kl_regress_top_percentile_invalid(self): + data, label = get_data_label(fetch_california_housing()) + data = data.drop(columns=["Latitude", "Longitude", "Population"]) + + method = SelectionMethod.Statistical(num_features=0.6, method="kl_divergence") + selector = Selective(method) + with self.assertRaises(TypeError): + selector.fit(data, label) + + def test_kl_classif_top_k(self): + data, label = get_data_label(load_iris()) + + method = SelectionMethod.Statistical(num_features=2, method="kl_divergence") + selector = Selective(method) + selector.fit(data, label) + subset = selector.transform(data) + + # Reduced columns + self.assertEqual(subset.shape[1], 2) + self.assertListEqual(list(subset.columns), ['petal length (cm)', 'petal width (cm)']) + + def test_kl_classif_top_percentile(self): + data, label = get_data_label(load_iris()) + + method = SelectionMethod.Statistical(num_features=0.5, method="kl_divergence") + selector = Selective(method) + selector.fit(data, label) + subset = selector.transform(data) + + # Reduced columns + self.assertEqual(subset.shape[1], 2) + self.assertListEqual(list(subset.columns), ['petal length (cm)', 'petal width (cm)']) + + def test_kl_classif_top_percentile_all(self): + data, label = get_data_label(load_iris()) + + method = SelectionMethod.Statistical(num_features=1.0, method="kl_divergence") + selector = Selective(method) + selector.fit(data, label) + subset = selector.transform(data) + + # Reduced columns + self.assertEqual(subset.shape[1], 4) + self.assertListEqual(list(subset.columns), + ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']) + + def test_kl_classif_top_k_all(self): + data, label = get_data_label(load_iris()) + + method = SelectionMethod.Statistical(num_features=4, method="kl_divergence") + selector = Selective(method) + selector.fit(data, label) + subset = selector.transform(data) + + # Reduced columns + self.assertEqual(subset.shape[1], 4) + self.assertListEqual(list(subset.columns), + ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']) From 77f86dd8a78eb60ed36f2b24396a0a7090857a89 Mon Sep 17 00:00:00 2001 From: zohairshafi Date: Mon, 11 Aug 2025 12:36:14 -0400 Subject: [PATCH 05/10] KL Test Updates --- feature/kl_divergence.py | 5 +++-- tests/run_all.py | 1 - tests/test_stat_kl.py | 22 +++++++++++++++++++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/feature/kl_divergence.py b/feature/kl_divergence.py index 06c9d50..fd0593e 100644 --- a/feature/kl_divergence.py +++ b/feature/kl_divergence.py @@ -24,14 +24,15 @@ def fit(self, X: pd.DataFrame, y: pd.Series) -> NoReturn: label_categories = np.unique(y) check_true(len(label_categories) == 2, TypeError("Only binary labels are supported for KL Divergence")) + input_dimension = X.shape[1] - kl_mat = np.zeros((self.num_features, 1)) + kl_mat = np.zeros((input_dimension, 1)) X = X.values class_one_idx = np.where(y == label_categories[0])[0] class_two_idx = np.where(y == label_categories[1])[0] - for i in range(self.num_features): + for i in range(input_dimension): f1 = np.histogram(X[class_one_idx, i], bins = self.num_bins)[0] f2 = np.histogram(X[class_two_idx, i], bins = self.num_bins)[0] diff --git a/tests/run_all.py b/tests/run_all.py index 2079a9c..8cd7089 100644 --- a/tests/run_all.py +++ b/tests/run_all.py @@ -5,7 +5,6 @@ import unittest - # Test Directory start_dir = '.' diff --git a/tests/test_stat_kl.py b/tests/test_stat_kl.py index d1aa506..e7974dd 100644 --- a/tests/test_stat_kl.py +++ b/tests/test_stat_kl.py @@ -31,6 +31,10 @@ def test_kl_regress_top_percentile_invalid(self): def test_kl_classif_top_k(self): data, label = get_data_label(load_iris()) + # Only Binary Data Is Supported by KL Divergence + data = data[(label == 0) | (label == 1)] + label = label[(label == 0) | (label == 1)] + method = SelectionMethod.Statistical(num_features=2, method="kl_divergence") selector = Selective(method) selector.fit(data, label) @@ -38,11 +42,15 @@ def test_kl_classif_top_k(self): # Reduced columns self.assertEqual(subset.shape[1], 2) - self.assertListEqual(list(subset.columns), ['petal length (cm)', 'petal width (cm)']) + self.assertListEqual(list(subset.columns), ['sepal length (cm)', 'sepal width (cm)']) def test_kl_classif_top_percentile(self): data, label = get_data_label(load_iris()) + # Only Binary Data Is Supported by KL Divergence + data = data[(label == 0) | (label == 1)] + label = label[(label == 0) | (label == 1)] + method = SelectionMethod.Statistical(num_features=0.5, method="kl_divergence") selector = Selective(method) selector.fit(data, label) @@ -50,10 +58,14 @@ def test_kl_classif_top_percentile(self): # Reduced columns self.assertEqual(subset.shape[1], 2) - self.assertListEqual(list(subset.columns), ['petal length (cm)', 'petal width (cm)']) + self.assertListEqual(list(subset.columns), ['sepal length (cm)', 'petal length (cm)']) def test_kl_classif_top_percentile_all(self): data, label = get_data_label(load_iris()) + + # Only Binary Data Is Supported by KL Divergence + data = data[(label == 0) | (label == 1)] + label = label[(label == 0) | (label == 1)] method = SelectionMethod.Statistical(num_features=1.0, method="kl_divergence") selector = Selective(method) @@ -68,6 +80,10 @@ def test_kl_classif_top_percentile_all(self): def test_kl_classif_top_k_all(self): data, label = get_data_label(load_iris()) + # Only Binary Data Is Supported by KL Divergence + data = data[(label == 0) | (label == 1)] + label = label[(label == 0) | (label == 1)] + method = SelectionMethod.Statistical(num_features=4, method="kl_divergence") selector = Selective(method) selector.fit(data, label) @@ -76,4 +92,4 @@ def test_kl_classif_top_k_all(self): # Reduced columns self.assertEqual(subset.shape[1], 4) self.assertListEqual(list(subset.columns), - ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']) + ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']) \ No newline at end of file From 3a449d1128af1d33a8de1d202cf1e4a8dd1b7af6 Mon Sep 17 00:00:00 2001 From: zohairshafi Date: Mon, 11 Aug 2025 12:43:01 -0400 Subject: [PATCH 06/10] Comments --- feature/kl_divergence.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/feature/kl_divergence.py b/feature/kl_divergence.py index fd0593e..a90f641 100644 --- a/feature/kl_divergence.py +++ b/feature/kl_divergence.py @@ -34,9 +34,11 @@ def fit(self, X: pd.DataFrame, y: pd.Series) -> NoReturn: for i in range(input_dimension): + # Create two distributions, one for the positive label and one for the negative label f1 = np.histogram(X[class_one_idx, i], bins = self.num_bins)[0] f2 = np.histogram(X[class_two_idx, i], bins = self.num_bins)[0] - + + # Normalize the distributions to be between 0 and 1 f1 = f1 / np.sum(f1) f2 = f2 / np.sum(f2) @@ -44,9 +46,13 @@ def fit(self, X: pd.DataFrame, y: pd.Series) -> NoReturn: kl = rel_entr(f1, f2) kl_reversed = rel_entr(f2, f1) + # The relative entropy function returns KL(P || Q) = np.inf when P == 0 and Q != 0. kl[kl == np.inf] = 9999 kl_reversed[kl_reversed == np.inf] = 9999 - + + # The final score is the combination of KL divergence in both directions. + # This could possibly be a flag in a future version to determine which direction to apply KL Divergence + # in if bidirectional is not desired. kl_mat[i] = np.sum(kl) + np.sum(kl_reversed) scores_ = kl_mat.flatten() From 2d802ec5130a48a93df58d221936ce5e06833b4f Mon Sep 17 00:00:00 2001 From: Ashish Peruri Date: Thu, 28 Aug 2025 09:53:31 -0400 Subject: [PATCH 07/10] update kl divergence test --- tests/test_stat_kl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_stat_kl.py b/tests/test_stat_kl.py index e7974dd..423821f 100644 --- a/tests/test_stat_kl.py +++ b/tests/test_stat_kl.py @@ -42,7 +42,7 @@ def test_kl_classif_top_k(self): # Reduced columns self.assertEqual(subset.shape[1], 2) - self.assertListEqual(list(subset.columns), ['sepal length (cm)', 'sepal width (cm)']) + self.assertListEqual(list(subset.columns), ['sepal length (cm)', 'petal length (cm)']) def test_kl_classif_top_percentile(self): data, label = get_data_label(load_iris()) From 556941d9be1d29fb627aad9a004de447c1bd4d93 Mon Sep 17 00:00:00 2001 From: Ashish Peruri Date: Thu, 28 Aug 2025 10:12:45 -0400 Subject: [PATCH 08/10] update setup and ci --- .github/workflows/ci.yml | 6 +++++- setup.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87f3158..d787325 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - + - name: Install OpenMP runtime (macOS) + if: runner.os == 'macOS' + run: | + brew update + brew install libomp - name: Check shell: bash run: | diff --git a/setup.py b/setup.py index 5c7bc4c..e0239a1 100644 --- a/setup.py +++ b/setup.py @@ -26,10 +26,10 @@ packages=setuptools.find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), classifiers=[ "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Operating System :: OS Independent", ], project_urls={"Source": "https://github.com/fidelity/selective"}, install_requires=required, - python_requires=">=3.7" + python_requires=">=3.8" ) From c5d429a3c4e1f050f72f3c9b82998f3dcaa92008 Mon Sep 17 00:00:00 2001 From: Ashish Peruri Date: Thu, 28 Aug 2025 16:31:49 -0400 Subject: [PATCH 09/10] update ci for macos --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d787325..db74f39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: if: runner.os == 'macOS' run: | brew update - brew install libomp + brew install libomp cbc - name: Check shell: bash run: | From b704d69f14e749bab2552dcbb8f30034debb8b7c Mon Sep 17 00:00:00 2001 From: Ashish Peruri Date: Fri, 29 Aug 2025 13:17:28 -0400 Subject: [PATCH 10/10] update readme and ci --- .github/workflows/ci.yml | 7 +------ README.md | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db74f39..d920820 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest] fail-fast: false steps: - uses: actions/checkout@v2 @@ -24,11 +24,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install OpenMP runtime (macOS) - if: runner.os == 'macOS' - run: | - brew update - brew install libomp cbc - name: Check shell: bash run: | diff --git a/README.md b/README.md index a07ba76..d27159b 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ plot_importance(df) ## Installation -Selective requires **Python 3.7+** and can be installed from PyPI using ``pip install selective``. +Selective requires **Python 3.8+** and can be installed from PyPI using ``pip install selective``. ## Source