From e2b997badf8f9d9d207839b90abc2c9b7207fdb8 Mon Sep 17 00:00:00 2001 From: Tanushree Tunstall Date: Sat, 5 Mar 2022 15:13:43 +0000 Subject: [PATCH] trying feature selection for classification logistic algorithm on 3 types of target --- my_data7.py | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 my_data7.py diff --git a/my_data7.py b/my_data7.py new file mode 100644 index 0000000..220148d --- /dev/null +++ b/my_data7.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Sat Mar 5 12:57:32 2022 + +@author: tanu +""" + +import os, sys +import pandas as pd +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.naive_bayes import BernoulliNB +from sklearn.neighbors import KNeighborsClassifier +from sklearn.svm import SVC +from sklearn.tree import DecisionTreeClassifier +from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier +from sklearn.neural_network import MLPClassifier +from sklearn.pipeline import Pipeline +from xgboost import XGBClassifier +from sklearn.preprocessing import StandardScaler +from sklearn.preprocessing import MinMaxScaler +from sklearn.model_selection import train_test_split +from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, roc_auc_score, roc_curve, f1_score +from sklearn.model_selection import cross_validate +from sklearn.metrics import make_scorer +from sklearn.metrics import classification_report + +from sklearn.feature_selection import RFE +from sklearn.feature_selection import RFECV + +#%% +model= Pipeline(steps = [ + ('pre', MinMaxScaler()), + ('reg', LogisticRegression(class_weight = 'balanced'))]) + +def precision(y_true,y_pred): + return precision_score(y_true,y_pred,pos_label = 1) +def recall(y_true,y_pred): + return recall_score(y_true, y_pred, pos_label = 1) +def f1(y_true,y_pred): + return f1_score(y_true, y_pred, pos_label = 1) + +acc = make_scorer(accuracy_score) +prec = make_scorer(precision) +rec = make_scorer(recall) +f1 = make_scorer(f1) + +output = cross_validate(model, X_train, y_train + , scoring = {'acc' : acc + ,'prec': prec + ,'rec' : rec + ,'f1' : f1} + , cv = 10 + , return_train_score = False) +pd.DataFrame(output).mean() + +#%% +# classification_repor: lowest scores but does it give numbers for all your classes! +model.fit(X_train, y_train) +y_pred = model.predict(X_test) + +f1_score(y_test, y_pred) +roc_auc_score (y_test, y_pred) # not sure! +#roc_curve(y_test, y_pred) +classification_report(y_test, y_pred) +target_names = {1:'Resistant', 0:'Sensitive'} +print(classification_report(y_test + , y_pred + #, target_names=y_test.map(target_names) + )) +#%%NOT SURE! +from itertools import combinations +def train(X): + return cross_validate(model, X, y_train + #, scoring = make_scorer(accuracy_score) + , scoring = {'acc' : acc + ,'prec' : prec + ,'rec' : rec + ,'f1' : f1} + , cv = 10 + , return_train_score = False) + #, return_estimator = True)['test_score'] + +scores = [train(X_train.loc[:,vars]) for vars in combinations(X_train.columns, len(X_train.columns))] +means = [score.mean() for score in scores] +means +#%% +# TO TRY +https://rasbt.github.io/mlxtend/ + +# stackoverflow +# informative post +https://datascience.stackexchange.com/questions/937/does-scikit-learn-have-a-forward-selection-stepwise-regression-algorithm +https://datascience.stackexchange.com/questions/24405/how-to-do-stepwise-regression-using-sklearn/24447#24447 +https://stats.stackexchange.com/questions/204141/difference-between-selecting-features-based-on-f-regression-and-based-on-r2 + + +# 0.24 version, it supports +https://scikit-learn.org/stable/auto_examples/release_highlights/plot_release_highlights_0_24_0.html#new-sequentialfeatureselector-transformer +https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.SelectFromModel.html +https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.SequentialFeatureSelector.html +https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.RFECV.html +https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_regression.html +https://www.scikit-yb.org/en/latest/api/model_selection/rfecv.html +#%% RFE: Feature selection in classification +# others in example +# https://towardsdatascience.com/feature-selection-techniques-for-classification-and-python-tips-for-their-application-10c0ddd7918b +# https://towardsdatascience.com/feature-selection-using-python-for-classification-problem-b5f00a1c7028 + +model_logistic = LogisticRegression(solver='lbfgs' + , multi_class = 'multinomial' + , max_iter = 1000) +model_logistic = LogisticRegression() + +sel_rfe_logistic = RFE(estimator = model_logistic + , n_features_to_select = 4 + , step = 1) + +X_train_rfe_logistic = sel_rfe_logistic.fit_transform(X_train, y_train) +print(sel_rfe_logistic.get_support()) + +print(sel_rfe_logistic.ranking_) +#%% RFECV +# https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.RFECV.html + +target = target1 +target = target3 +target = target4 +X_train, X_test, y_train, y_test = train_test_split(X_vars1, + target, + test_size = 0.33, + random_state = 42) + +X_train, X_test, y_train, y_test = train_test_split(X_vars2, + target, + test_size = 0.33, + random_state = 42) + +X_train, X_test, y_train, y_test = train_test_split(X_vars3, + target, + test_size = 0.33, + random_state = 42) + +X_train, X_test, y_train, y_test = train_test_split(X_vars5, + target, + test_size = 0.33, + random_state = 42) + +X_train, X_test, y_train, y_test = train_test_split(X_vars11, + target, + test_size = 0.33, + random_state = 42) + +model_logistic = LogisticRegression() +sel_rfe_logistic = RFECV(estimator = model_logistic + , cv = 10 + , step = 1) + +X_train_rfe_logistic = sel_rfe_logistic.fit_transform(X_train, y_train) +print(sel_rfe_logistic.get_support()) +X_train.columns + +print(sel_rfe_logistic.ranking_)