diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87f3158..d920820 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest] fail-fast: false steps: - uses: actions/checkout@v2 @@ -24,7 +24,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Check shell: bash run: | diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 48a1b4a..2a42bb8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,12 @@ CHANGELOG ========= +------------------------------------------------------------------------------- +August 7, 2025 1.2.0 +------------------------------------------------------------------------------- + +- Added KL Divergence based feature selection for binary labels. Thanks to @zohairshafi for contributing this method. + ------------------------------------------------------------------------------- April, 24, 2023 1.1.2 ------------------------------------------------------------------------------- diff --git a/README.md b/README.md index f4b0bcb..d27159b 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ print("Scores:", list(selector.get_absolute_scores())) |:--------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| | [Variance per Feature](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.VarianceThreshold.html) | `threshold` | | [Correlation pairwise Features](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.corr.html) | [Pearson Correlation Coefficient](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient)
[Kendall Rank Correlation Coefficient](https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient)
[Spearman's Rank Correlation Coefficient](https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient)
| -| [Statistical Analysis](https://scikit-learn.org/stable/modules/feature_selection.html#univariate-feature-selection) | [ANOVA F-test Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_classif.html)
[F-value Regression](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_regression.html)
[Chi-Square](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.chi2.html)
[Mutual Information Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.mutual_info_classif.html)
[Variance Inflation Factor](https://www.statsmodels.org/stable/generated/statsmodels.stats.outliers_influence.variance_inflation_factor.html) | +| [Statistical Analysis](https://scikit-learn.org/stable/modules/feature_selection.html#univariate-feature-selection) | [ANOVA F-test Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_classif.html)
[F-value Regression](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_regression.html)
[Chi-Square](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.chi2.html)
[KL Divergence](https://en.wikipedia.org/wiki/Kullback–Leibler_divergence)
[Mutual Information Classification](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.mutual_info_classif.html)
[Variance Inflation Factor](https://www.statsmodels.org/stable/generated/statsmodels.stats.outliers_influence.variance_inflation_factor.html) | | [Linear Methods](https://en.wikipedia.org/wiki/Linear_regression) | [Linear Regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html?highlight=linear%20regression#sklearn.linear_model.LinearRegression)
[Logistic Regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html?highlight=logistic%20regression#sklearn.linear_model.LogisticRegression)
[Lasso Regularization](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html#sklearn.linear_model.Lasso)
[Ridge Regularization](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge)
| | [Tree-based Methods](https://scikit-learn.org/stable/modules/tree.html) | [Decision Tree](https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html#sklearn.tree.DecisionTreeClassifier)
[Random Forest](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html?highlight=random%20forest#sklearn.ensemble.RandomForestClassifier)
[Extra Trees Classifier](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html)
[XGBoost](https://xgboost.readthedocs.io/en/latest/)
[LightGBM](https://lightgbm.readthedocs.io/en/latest/)
[AdaBoost](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html)
[CatBoost](https://github.com/catboost)
[Gradient Boosting Tree](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html)
| | [Text-based Methods](https://link.springer.com/chapter/10.1007/978-3-030-78230-6_27) | `featurization_method` = [TextWiser](https://github.com/fidelity/textwiser)
`optimization_method = ["exact", "greedy", "kmeans", "random"]`
`cost_metric = ["unicost", "diverse"]` | @@ -81,6 +81,7 @@ selectors = { # Statistical methods "stat_anova": SelectionMethod.Statistical(num_features, method="anova"), "stat_chi_square": SelectionMethod.Statistical(num_features, method="chi_square"), + "stat_kl_divergence": SelectionMethod.Statistical(num_features, method="kl_divergence"), "stat_mutual_info": SelectionMethod.Statistical(num_features, method="mutual_info"), # Linear methods @@ -168,7 +169,7 @@ plot_importance(df) ## Installation -Selective requires **Python 3.7+** and can be installed from PyPI using ``pip install selective``. +Selective requires **Python 3.8+** and can be installed from PyPI using ``pip install selective``. ## Source diff --git a/feature/_version.py b/feature/_version.py index 35e9165..0f3c50c 100644 --- a/feature/_version.py +++ b/feature/_version.py @@ -2,4 +2,4 @@ # Copyright FMR LLC # SPDX-License-Identifier: Apache-2.0 -__version__ = "1.1.2" \ No newline at end of file +__version__ = "1.2.0" \ No newline at end of file diff --git a/feature/kl_divergence.py b/feature/kl_divergence.py new file mode 100644 index 0000000..a90f641 --- /dev/null +++ b/feature/kl_divergence.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright FMR LLC +# SPDX-License-Identifier: Apache-2.0 + +from typing import NoReturn, Tuple + +import pandas as pd +import numpy as np + +from scipy.special import rel_entr +from feature.base import _BaseSupervisedSelector +from feature.utils import Num, check_true + + +class _KL_Divergence(_BaseSupervisedSelector): + + def __init__(self, seed: int, num_features: Num, num_bins: Num = 100): + super().__init__(seed) + + self.num_features = num_features # this could be int or float + self.num_bins = num_bins + + def fit(self, X: pd.DataFrame, y: pd.Series) -> NoReturn: + + label_categories = np.unique(y) + check_true(len(label_categories) == 2, TypeError("Only binary labels are supported for KL Divergence")) + input_dimension = X.shape[1] + + kl_mat = np.zeros((input_dimension, 1)) + X = X.values + + class_one_idx = np.where(y == label_categories[0])[0] + class_two_idx = np.where(y == label_categories[1])[0] + + for i in range(input_dimension): + + # Create two distributions, one for the positive label and one for the negative label + f1 = np.histogram(X[class_one_idx, i], bins = self.num_bins)[0] + f2 = np.histogram(X[class_two_idx, i], bins = self.num_bins)[0] + + # Normalize the distributions to be between 0 and 1 + f1 = f1 / np.sum(f1) + f2 = f2 / np.sum(f2) + + # KL Divergence is not symmetric, so we calculate divergence in both directions + kl = rel_entr(f1, f2) + kl_reversed = rel_entr(f2, f1) + + # The relative entropy function returns KL(P || Q) = np.inf when P == 0 and Q != 0. + kl[kl == np.inf] = 9999 + kl_reversed[kl_reversed == np.inf] = 9999 + + # The final score is the combination of KL divergence in both directions. + # This could possibly be a flag in a future version to determine which direction to apply KL Divergence + # in if bidirectional is not desired. + kl_mat[i] = np.sum(kl) + np.sum(kl_reversed) + + scores_ = kl_mat.flatten() + + self.scores_ = scores_ # This is used by the statistical.py fit function. + self.abs_scores = scores_ + + def transform(self, data: pd.DataFrame) -> pd.DataFrame: + + # Select top-k from data based on abs_scores and num_features + return self.get_top_k(data, self.abs_scores) diff --git a/feature/selector.py b/feature/selector.py index 80cf152..75f213e 100644 --- a/feature/selector.py +++ b/feature/selector.py @@ -36,7 +36,7 @@ __author__ = "FMR LLC" -__version__ = "1.0.0" +__version__ = "1.2.0" __copyright__ = "Copyright (C), FMR LLC" @@ -202,6 +202,13 @@ class Statistical(NamedTuple): searching for the optimal binning strategy. Note: MIC is dropped from Selective due to inactive MINE library + The KL Divergence feature importance should only be used with + binary labels. It computes the distribution of a given feature for instances where label == 1 and label == 0. + Uses KL divergence between the two distributions as an importance score, + where a higher value indicates greater discriminative power of the feature + with respect to the binary label. Since KL Divergence is non-symmetric, this method + computer the divergence in both directions and sums them up. + Notes on Randomness: - Mutual Info is non-deterministic, depends on the seed value. - The other methods are deterministic @@ -227,7 +234,7 @@ def _validate(self): if isinstance(self.num_features, float): check_true(self.num_features <= 1, ValueError("Num features ratio must be between [0..1].")) # "maximal_info" dropped - check_true(self.method in ["anova", "chi_square", "mutual_info", "variance_inflation"], + check_true(self.method in ["anova", "chi_square", "kl_divergence", "mutual_info", "variance_inflation"], ValueError("Statistical method can only be anova, chi_square, or mutual_info.")) class TreeBased(NamedTuple): diff --git a/feature/statistical.py b/feature/statistical.py index fec4844..3f4288b 100644 --- a/feature/statistical.py +++ b/feature/statistical.py @@ -10,9 +10,11 @@ import pandas as pd from sklearn.feature_selection import chi2, f_classif, f_regression, mutual_info_classif, mutual_info_regression from statsmodels.stats.outliers_influence import variance_inflation_factor +from scipy.special import rel_entr from feature.base import _BaseSupervisedSelector, _BaseDispatcher from feature.utils import get_selector, Num, get_task_string +from feature.kl_divergence import _KL_Divergence class _Statistical(_BaseSupervisedSelector, _BaseDispatcher): @@ -33,15 +35,17 @@ def __init__(self, seed: int, num_features: Num, method: str): self.imp = None # Implementor factory - self.factory = {"regression_anova": f_regression, - "regression_chi_square": None, - "regression_mutual_info": partial(mutual_info_regression, random_state=self.seed), - # "regression_maximal_info": MINE(), # dropped - "classification_anova": f_classif, + self.factory = {"classification_anova": f_classif, "classification_chi_square": chi2, "classification_mutual_info": partial(mutual_info_classif, random_state=self.seed), # "classification_maximal_info": MINE(), # dropped - "unsupervised_variance_inflation": variance_inflation_factor} + "kl_divergence" : _KL_Divergence(num_features = self.num_features, seed = self.seed), + "regression_anova": f_regression, + "regression_chi_square": None, + "regression_mutual_info": partial(mutual_info_regression, random_state=self.seed), + # "regression_maximal_info": MINE(), # dropped + "unsupervised_variance_inflation": variance_inflation_factor, + } def get_model_args(self, selection_method) -> Tuple: @@ -54,7 +58,9 @@ def dispatch_model(self, labels: pd.Series, *args): method = args[0] # Get statistical scoring function - if method == "variance_inflation": + if method == "kl_divergence": + score_func = self.factory.get(method) + elif method == "variance_inflation": score_func = self.factory.get("unsupervised_" + method) else: score_func = self.factory.get(get_task_string(labels) + method) @@ -62,6 +68,8 @@ def dispatch_model(self, labels: pd.Series, *args): # Check scoring compatibility with task if score_func is None: raise TypeError(method + " cannot be used for task: " + get_task_string(labels)) + elif method == "kl_divergence": + self.imp = score_func elif method == "variance_inflation": # or isinstance(score_func, MINE) (dropped) self.imp = score_func else: @@ -82,6 +90,7 @@ def fit(self, data: pd.DataFrame, labels: pd.Series) -> NoReturn: if self.method == "variance_inflation": # VIF is unsupervised, regression between data and each feature self.abs_scores = np.array([variance_inflation_factor(data.values, i) for i in range(data.shape[1])]) + else: # sklearn selector model self.imp.fit(X=data, y=labels) diff --git a/setup.py b/setup.py index 5c7bc4c..e0239a1 100644 --- a/setup.py +++ b/setup.py @@ -26,10 +26,10 @@ packages=setuptools.find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), classifiers=[ "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Operating System :: OS Independent", ], project_urls={"Source": "https://github.com/fidelity/selective"}, install_requires=required, - python_requires=">=3.7" + python_requires=">=3.8" ) diff --git a/tests/run_all.py b/tests/run_all.py index 2079a9c..8cd7089 100644 --- a/tests/run_all.py +++ b/tests/run_all.py @@ -5,7 +5,6 @@ import unittest - # Test Directory start_dir = '.' diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 149da2c..a795ea8 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -29,6 +29,7 @@ class TestBenchmark(BaseTest): "univ_anova": SelectionMethod.Statistical(num_features, method="anova"), "univ_chi_square": SelectionMethod.Statistical(num_features, method="chi_square"), "univ_mutual_info": SelectionMethod.Statistical(num_features, method="mutual_info"), + "kl_divergence": SelectionMethod.Statistical(num_features, method="kl_divergence"), "linear": SelectionMethod.Linear(num_features, regularization="none"), "lasso": SelectionMethod.Linear(num_features, regularization="lasso", alpha=alpha), "ridge": SelectionMethod.Linear(num_features, regularization="ridge", alpha=alpha), diff --git a/tests/test_stat_kl.py b/tests/test_stat_kl.py new file mode 100644 index 0000000..423821f --- /dev/null +++ b/tests/test_stat_kl.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Copyright FMR LLC +# SPDX-License-Identifier: Apache-2.0 + +from sklearn.datasets import fetch_california_housing, load_iris +from feature.utils import get_data_label +from feature.selector import Selective, SelectionMethod +from tests.test_base import BaseTest + + +class TestKL(BaseTest): + + def test_kl_regress_invalid(self): + data, label = get_data_label(fetch_california_housing()) + data = data.drop(columns=["Latitude", "Longitude", "Population"]) + + method = SelectionMethod.Statistical(num_features=3, method="kl_divergence") + selector = Selective(method) + with self.assertRaises(TypeError): + selector.fit(data, label) + + def test_kl_regress_top_percentile_invalid(self): + data, label = get_data_label(fetch_california_housing()) + data = data.drop(columns=["Latitude", "Longitude", "Population"]) + + method = SelectionMethod.Statistical(num_features=0.6, method="kl_divergence") + selector = Selective(method) + with self.assertRaises(TypeError): + selector.fit(data, label) + + def test_kl_classif_top_k(self): + data, label = get_data_label(load_iris()) + + # Only Binary Data Is Supported by KL Divergence + data = data[(label == 0) | (label == 1)] + label = label[(label == 0) | (label == 1)] + + method = SelectionMethod.Statistical(num_features=2, method="kl_divergence") + selector = Selective(method) + selector.fit(data, label) + subset = selector.transform(data) + + # Reduced columns + self.assertEqual(subset.shape[1], 2) + self.assertListEqual(list(subset.columns), ['sepal length (cm)', 'petal length (cm)']) + + def test_kl_classif_top_percentile(self): + data, label = get_data_label(load_iris()) + + # Only Binary Data Is Supported by KL Divergence + data = data[(label == 0) | (label == 1)] + label = label[(label == 0) | (label == 1)] + + method = SelectionMethod.Statistical(num_features=0.5, method="kl_divergence") + selector = Selective(method) + selector.fit(data, label) + subset = selector.transform(data) + + # Reduced columns + self.assertEqual(subset.shape[1], 2) + self.assertListEqual(list(subset.columns), ['sepal length (cm)', 'petal length (cm)']) + + def test_kl_classif_top_percentile_all(self): + data, label = get_data_label(load_iris()) + + # Only Binary Data Is Supported by KL Divergence + data = data[(label == 0) | (label == 1)] + label = label[(label == 0) | (label == 1)] + + method = SelectionMethod.Statistical(num_features=1.0, method="kl_divergence") + selector = Selective(method) + selector.fit(data, label) + subset = selector.transform(data) + + # Reduced columns + self.assertEqual(subset.shape[1], 4) + self.assertListEqual(list(subset.columns), + ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']) + + def test_kl_classif_top_k_all(self): + data, label = get_data_label(load_iris()) + + # Only Binary Data Is Supported by KL Divergence + data = data[(label == 0) | (label == 1)] + label = label[(label == 0) | (label == 1)] + + method = SelectionMethod.Statistical(num_features=4, method="kl_divergence") + selector = Selective(method) + selector.fit(data, label) + subset = selector.transform(data) + + # Reduced columns + self.assertEqual(subset.shape[1], 4) + self.assertListEqual(list(subset.columns), + ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']) \ No newline at end of file