added all classification algorithms params for gridsearch
This commit is contained in:
parent
d012542435
commit
0c4f1e1e5f
8 changed files with 503 additions and 110 deletions
Binary file not shown.
|
|
@ -15,6 +15,8 @@ Created on Fri Mar 18 09:47:48 2022
|
|||
# https://uk.mathworks.com/help/stats/hyperparameter-optimization-in-classification-learner-app.html [ algo]
|
||||
# As a general rule of thumb, it is required to run baseline models on the dataset. I know H2O- AutoML and other AutoML packages do this. But I want to try using Scikit-learn Pipeline,
|
||||
# https://codereview.stackexchange.com/questions/256934/model-pipeline-to-run-multiple-classifiers-for-ml-classification
|
||||
# https://uk.mathworks.com/help/stats/hyperparameter-optimization-in-classification-learner-app.html
|
||||
# QDA: https://www.geeksforgeeks.org/quadratic-discriminant-analysis/
|
||||
|
||||
names = [
|
||||
"Nearest Neighbors",
|
||||
|
|
@ -41,3 +43,222 @@ classifiers = [
|
|||
GaussianNB(),
|
||||
QuadraticDiscriminantAnalysis(),
|
||||
]
|
||||
|
||||
# NOTE Logistic regression
|
||||
# The choice of the algorithm depends on the penalty chosen: Supported penalties by solver:
|
||||
# ‘newton-cg’ - [‘l2’, ‘none’]
|
||||
# ‘lbfgs’ - [‘l2’, ‘none’]
|
||||
# ‘liblinear’ - [‘l1’, ‘l2’]
|
||||
# ‘sag’ - [‘l2’, ‘none’]
|
||||
# ‘saga’ - [‘elasticnet’, ‘l1’, ‘l2’, ‘none’]
|
||||
|
||||
# SVR?
|
||||
# estimator=SVR(kernel='rbf')
|
||||
# param_grid={
|
||||
# 'C': [1.1, 5.4, 170, 1001],
|
||||
# 'epsilon': [0.0003, 0.007, 0.0109, 0.019, 0.14, 0.05, 8, 0.2, 3, 2, 7],
|
||||
# 'gamma': [0.7001, 0.008, 0.001, 3.1, 1, 1.3, 5]
|
||||
# }
|
||||
|
||||
|
||||
#%% Classification algorithms param grid
|
||||
#%% LogisticRegression()
|
||||
#https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
|
||||
gs_lr = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
,('clf', LogisticRegression(**rs
|
||||
, **njobs))
|
||||
))
|
||||
gs_lr_params = {
|
||||
'clf__C' : [0.0001, 0.001, 0.01, 0.1 ,1, 10, 100]
|
||||
#'C': np.logspace(-4, 4, 50)
|
||||
, 'clf__penalty': ['l1', 'l2', 'elasticnet', 'none']
|
||||
, 'clf__solver' : ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']
|
||||
}
|
||||
#%% DecisionTreeClassifier()
|
||||
|
||||
gs_dt = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
, ('clf', DecisionTreeClassifier(**rs
|
||||
, **njobs))
|
||||
))
|
||||
gs_dt_params = {
|
||||
'clf__max_depth': [ 2, 4, 6, 8, 10]
|
||||
, 'clf__criterion':['gini','entropy']
|
||||
, "clf__max_features":["auto", None]
|
||||
, "clf__max_leaf_nodes":[10,20,30,40]
|
||||
}
|
||||
#%% KNeighborsClassifier()
|
||||
#https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html
|
||||
gs_knn = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
,('clf', KNeighborsClassifier(**rs
|
||||
, **njobs))
|
||||
))
|
||||
|
||||
gs_knn_params = {
|
||||
'clf__n_neighbors': [3, 7, 10]
|
||||
#, 'clf__n_neighbors': range(1, 21, 2)
|
||||
,'clf__metric' : ['euclidean', 'manhattan', 'minkowski']
|
||||
, 'clf__weights' : ['uniform', 'distance']
|
||||
}
|
||||
#%% RandomForestClassifier()
|
||||
|
||||
gs_rf = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
,('clf', RandomForestClassifier(**rs
|
||||
, **njobs
|
||||
, bootstrap = True
|
||||
, oob_score = True))
|
||||
))
|
||||
gs_rf_params = {
|
||||
'clf__max_depth': [4, 6, 8, 10, 12, 16, 20, None]
|
||||
, 'clf__class_weight':['balanced','balanced_subsample']
|
||||
, 'clf__n_estimators': [10, 100, 1000]
|
||||
, 'clf__criterion': ['gini', 'entropy']
|
||||
, 'clf__max_features': ['auto', 'sqrt']
|
||||
, 'clf__min_samples_leaf': [2, 4, 8, 50]
|
||||
, 'clf__min_samples_split': [10, 20]
|
||||
}
|
||||
#%% XGBClassifier()
|
||||
# https://stackoverflow.com/questions/34674797/xgboost-xgbclassifier-defaults-in-python
|
||||
# https://stackoverflow.com/questions/34674797/xgboost-xgbclassifier-defaults-in-python
|
||||
gs_xgb = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
,('clf', XGBClassifier(**rs
|
||||
, **njobs))
|
||||
))
|
||||
|
||||
gs_xgb_params = {
|
||||
'clf__learning_rate': [0.01, 0.05, 0.1, 0.2]
|
||||
, 'clf__max_depth': [4, 6, 8, 10, 12, 16, 20]
|
||||
, 'clf__min_samples_leaf': [4, 8, 12, 16, 20]
|
||||
, 'clf__max_features': ['auto', 'sqrt']
|
||||
}
|
||||
#%% MLPClassifier()
|
||||
# https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html
|
||||
gs_mlp = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
,('clf', MLPClassifier(**rs
|
||||
, **njobs
|
||||
, max_iter = 500))
|
||||
))
|
||||
|
||||
gs_mlp_params = {
|
||||
'clf__hidden_layer_sizes': [(1), (2), (3)]
|
||||
, 'clf__max_features': ['auto', 'sqrt']
|
||||
, 'clf__min_samples_leaf': [2, 4, 8]
|
||||
, 'clf__min_samples_split': [10, 20]
|
||||
}
|
||||
#%% RidgeClassifier()
|
||||
# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.RidgeClassifier.html
|
||||
gs_rc = Pipeline((
|
||||
('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
|
||||
,('clf', RidgeClassifier(**rs
|
||||
, **njobs))
|
||||
))
|
||||
|
||||
gs_rc_params = {
|
||||
'clf__alpha': [0.1, 0.2, 0.5, 0.8, 1.0]
|
||||
}
|
||||
|
||||
#%% SVC()
|
||||
# https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html
|
||||
gs_svc = Pipeline((
|
||||
('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
|
||||
,('clf', SVC(**rs
|
||||
, **njobs))
|
||||
))
|
||||
|
||||
gs_svc_params = {
|
||||
'clf__kernel': ['linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}
|
||||
, 'clf__C' : [50, 10, 1.0, 0.1, 0.01]
|
||||
, 'clf__gamma': ['scale', 'auto'] }
|
||||
|
||||
#%% BaggingClassifier()
|
||||
#https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.BaggingClassifier.html
|
||||
gs_bdt = Pipeline((
|
||||
('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
|
||||
,('clf', BaggingClassifier(**rs
|
||||
, **njobs
|
||||
, bootstrap = True
|
||||
, oob_score = True))
|
||||
))
|
||||
|
||||
gs_bdt_params = {
|
||||
'clf__n_estimators' : [10, 100, 1000]
|
||||
# If None, then the base estimator is a DecisionTreeClassifier.
|
||||
, 'clf__base_estimator' : ['None', 'SVC()', 'KNeighborsClassifier()']# if none, DT is used
|
||||
, 'clf__gamma': ['scale', 'auto'] }
|
||||
#%% GradientBoostingClassifier()
|
||||
# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html
|
||||
gs_gb = Pipeline((
|
||||
('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
|
||||
,('clf', GradientBoostingClassifier(**rs))
|
||||
))
|
||||
|
||||
gs_bdt_params = {
|
||||
'clf__n_estimators' : [10, 100, 1000]
|
||||
, 'clf__n_estimators' : [10, 100, 1000]
|
||||
, 'clf__learning_rate': [0.001, 0.01, 0.1]
|
||||
, 'clf__subsample' : [0.5, 0.7, 1.0]
|
||||
, 'clf__max_depth' : [3, 7, 9]
|
||||
}
|
||||
#%% AdaBoostClassifier()
|
||||
#https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html#sklearn.ensemble.AdaBoostClassifier
|
||||
gs_gb = Pipeline((
|
||||
('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
|
||||
,('clf', AdaBoostClassifier(**rs))
|
||||
))
|
||||
|
||||
gs_bdt_params = {
|
||||
'clf__n_estimators': [none, 1, 2]
|
||||
, 'clf__base_estiamtor' : ['None', 1*SVC(), 1*KNeighborsClassifier()]
|
||||
#, 'clf___splitter' : ["best", "random"]
|
||||
}
|
||||
#%% GaussianProcessClassifier()
|
||||
# https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.GaussianProcessClassifier.html
|
||||
#GaussianProcessClassifier(1.0 * RBF(1.0)),
|
||||
gs_gpc = Pipeline((
|
||||
('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
|
||||
,('clf', GaussianProcessClassifier(**rs))
|
||||
))
|
||||
|
||||
gs_gpc_params = {
|
||||
'clf__kernel': [1*RBF(), 1*DotProduct(), 1*Matern(), 1*RationalQuadratic(), 1*WhiteKernel()]
|
||||
}
|
||||
|
||||
#%% GaussianNB()
|
||||
# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html
|
||||
gs_gnb = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
, ('pca', PCA() )# CHECK if it wants -1 to 1
|
||||
,('clf', GaussianNB(**rs))
|
||||
))
|
||||
|
||||
gs_gnb_params = {
|
||||
'clf__priors': [None]
|
||||
, 'clf__var_smoothing': np.logspace(0,-9, num=100)
|
||||
}
|
||||
|
||||
#%% QuadraticDiscriminantAnalysis()
|
||||
#https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html
|
||||
|
||||
gs_qda = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
#, ('pca', PCA() )# CHECK if it wants -1 to 1
|
||||
,('clf', QuadraticDiscriminantAnalysis())
|
||||
))
|
||||
#%% BernoulliNB()
|
||||
# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.BernoulliNB.html
|
||||
gs_gnb = Pipeline((
|
||||
('pre' , MinMaxScaler())
|
||||
,('clf', BernoulliNB())
|
||||
))
|
||||
BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True)
|
||||
gs_gnb_params = {
|
||||
'clf__alpha': [0, 1]
|
||||
, 'clf__binarize':['None', 0]
|
||||
, 'clf__fit_prior': [True]
|
||||
, 'clf__class_prior': ['None']
|
||||
}
|
||||
108
hyperparams.py
108
hyperparams.py
|
|
@ -16,10 +16,9 @@ scoring_refit = {'scoring': mcc_score_fn
|
|||
,'refit': 'mcc'}
|
||||
#n_jobs = 10 # my desktop has 12 cores
|
||||
njobs = {'n_jobs': 10}
|
||||
skf_cv = StratifiedKFold(n_splits=10,shuffle = True)
|
||||
skf_cv = StratifiedKFold(n_splits = 10, shuffle = True)
|
||||
#cv = {'cv': 10}
|
||||
|
||||
|
||||
gs_dt = GridSearchCV(estimator=DecisionTreeClassifier(**rs
|
||||
#,class_weight = {1:10, 0:1}
|
||||
),
|
||||
|
|
@ -43,8 +42,8 @@ gs_dt_fit = gs_dt.fit(num_df_wtgt[numerical_FN]
|
|||
|
||||
gs_dt_fit_res = gs_dt_fit.cv_results_
|
||||
|
||||
print('Best model:\n', gs_dt.best_params_)
|
||||
print('Best models score:\n', gs_dt.best_score_)
|
||||
print('Best model:\n' , gs_dt.best_params_)
|
||||
print('Best models score:\n' , gs_dt.best_score_)
|
||||
print('Check best models score:\n', mean(gs_dt_fit_res['mean_test_mcc']))
|
||||
|
||||
#%% Check the scores:
|
||||
|
|
@ -106,3 +105,104 @@ means = clf.cv_results_['mean_test_score']
|
|||
stds = clf.cv_results_['std_test_score']
|
||||
for mean, std, params in zip(means, stds, clf.cv_results_['params']):
|
||||
print("%0.3f (+/-%0.03f) for %r" % (mean, std * 2, params))
|
||||
|
||||
########################################################################
|
||||
#%%: Hyperparams with SKF and trying different scoring functions
|
||||
# https://stackoverflow.com/questions/57248072/gridsearchcv-gives-different-result
|
||||
# https://stackoverflow.com/questions/44947574/what-is-the-meaning-of-mean-test-score-in-cv-result
|
||||
#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn
|
||||
|
||||
|
||||
#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn
|
||||
# If you only want accuracy, then you can simply use cross_val_score()
|
||||
# kf = KFold(n_splits=10)
|
||||
# clf_tree=DecisionTreeClassifier()
|
||||
# scores = cross_val_score(clf_tree, X, y, cv=kf)
|
||||
# avg_score = np.mean(score_array)
|
||||
# print(avg_score)
|
||||
# Here cross_val_score will take as input your original X and y (without splitting into train and test). cross_val_score will automatically split them into train and test, fit the model on train data and score on test data. And those scores will be returned in the scores variable.
|
||||
# So when you have 10 folds, 10 scores will be returned in scores variable. You can then just take an average of that.
|
||||
|
||||
mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
|
||||
|
||||
scoring_refit_recall = {'scoring': 'recall'
|
||||
,'refit': 'recall'}
|
||||
|
||||
scoring_refit_recall = {'scoring': 'precision'
|
||||
,'refit': 'precision'}
|
||||
|
||||
scoring_refit_mcc = {'scoring': mcc_score_fn
|
||||
,'refit': 'mcc'}
|
||||
#n_jobs = 10 # my desktop has 12 cores
|
||||
#cv = {'cv': 10}#%%
|
||||
|
||||
njobs = {'n_jobs': 10}
|
||||
skf_cv = StratifiedKFold(n_splits = 10, shuffle = True)
|
||||
|
||||
#%% GSCV: RandomForest
|
||||
gs_rf = GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, oob_score = True
|
||||
#,class_weight = {1: 10/11, 0: 1/11}
|
||||
)
|
||||
, param_grid=[{'max_depth': [4, 6, 8, 10, None]
|
||||
, 'max_features': ['auto', 'sqrt']
|
||||
, 'min_samples_leaf': [2, 4, 8]
|
||||
, 'min_samples_split': [10, 20]}]
|
||||
, cv = skf_cv
|
||||
, **njobs
|
||||
, **scoring_refit_recall
|
||||
#, **scoring_refit_mcc
|
||||
#, scoring = scoring_fn, refit = False
|
||||
)
|
||||
#gs_rf.fit(X_train, y_train)
|
||||
#gs_rf_fit = gs_rf.fit(X_train y_train)
|
||||
|
||||
gs_rf.fit(X, y)
|
||||
gs_rf_fit = gs_rf.fit(X, y)
|
||||
gs_rf_res = gs_rf_fit.cv_results_
|
||||
print('Best model:\n', gs_rf.best_params_)
|
||||
print('Best models score:\n', gs_rf.best_score_)
|
||||
print('Check mean models score:\n', mean(gs_rf_res['mean_test_score']))
|
||||
|
||||
#%% Proof of concept: manual inspection to see how best score is calcualted!
|
||||
# SATISFIED!
|
||||
# Best_model example: recall, Best model's score: 0.8059288537549408
|
||||
# {'max_depth': 4, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 10}
|
||||
|
||||
# Best model example: mcc, Best models score: 0.42504894661702863
|
||||
# {'max_depth': 4, 'max_features': 'auto', 'min_samples_leaf': 4, 'min_samples_split': 20}
|
||||
|
||||
# Best model example: precision, Best models score: 0.7144745254745255
|
||||
# {'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10}
|
||||
|
||||
best_model = [{'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10 }]
|
||||
|
||||
gs_results_df = pd.DataFrame(gs_rf_res)
|
||||
gs_results_df.shape
|
||||
|
||||
gs_best_df = gs_results_df.loc[gs_results_df['params'].isin(best_model)]
|
||||
gs_best_df.shape
|
||||
|
||||
gs_best_df_test = gs_best_df.filter(like = 'test_', axis = 1)
|
||||
gs_best_df_test.shape
|
||||
|
||||
gs_best_df_test_recall = gs_best_df_test.filter(like = '_score', axis = 1)
|
||||
gs_best_df_test_recall.shape
|
||||
|
||||
f = gs_best_df_test_recall.filter(like='split', axis = 1)
|
||||
f.shape
|
||||
#gs_best_df_test_mcc = gs_best_df_test.filter(like = '_mcc', axis = 1)
|
||||
#f = gs_best_df_test_mcc.filter(like='split', axis = 1)
|
||||
|
||||
f.iloc[:,[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]].mean(axis = 1)
|
||||
# recall: 0.801186 vs 0.8059288537549408
|
||||
# mcc: 0.425049 vs 0.42504894661702863
|
||||
# precision: 0.714475 vs 0.7144745254745255
|
||||
|
||||
#%%
|
||||
|
||||
#%% Check the scores:
|
||||
print([(len(train), len(test)) for train, test in skf_cv.split(X, y)])
|
||||
gs_rf_fit.cv_results_
|
||||
#its the weighted average!?
|
||||
#%%
|
||||
|
||||
|
|
|
|||
|
|
@ -1,106 +1,158 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Wed Mar 16 16:55:06 2022
|
||||
Created on Sun Mar 20 13:02:54 2022
|
||||
|
||||
@author: tanu
|
||||
"""
|
||||
# https://stackoverflow.com/questions/57248072/gridsearchcv-gives-different-result
|
||||
# https://stackoverflow.com/questions/44947574/what-is-the-meaning-of-mean-test-score-in-cv-result
|
||||
#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn
|
||||
# https://machinelearningmastery.com/hyperparameters-for-classification-machine-learning-algorithms/
|
||||
|
||||
#%% LogisticRegression
|
||||
# example of grid searching key hyperparametres for logistic regression
|
||||
from sklearn.datasets import make_blobs
|
||||
from sklearn.model_selection import RepeatedStratifiedKFold
|
||||
from sklearn.model_selection import GridSearchCV
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
# define dataset
|
||||
X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)
|
||||
# define models and parameters
|
||||
model = LogisticRegression()
|
||||
solvers = ['newton-cg', 'lbfgs', 'liblinear']
|
||||
penalty = ['l2']
|
||||
c_values = [100, 10, 1.0, 0.1, 0.01]
|
||||
# define grid search
|
||||
grid = dict(solver=solvers,penalty=penalty,C=c_values)
|
||||
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
|
||||
grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0)
|
||||
grid_result = grid_search.fit(X, y)
|
||||
# summarize results
|
||||
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
|
||||
means = grid_result.cv_results_['mean_test_score']
|
||||
stds = grid_result.cv_results_['std_test_score']
|
||||
params = grid_result.cv_results_['params']
|
||||
for mean, stdev, param in zip(means, stds, params):
|
||||
print("%f (%f) with: %r" % (mean, stdev, param))
|
||||
#%% RidgeClassifier
|
||||
from sklearn.datasets import make_blobs
|
||||
from sklearn.model_selection import RepeatedStratifiedKFold
|
||||
from sklearn.model_selection import GridSearchCV
|
||||
from sklearn.linear_model import RidgeClassifier
|
||||
# define dataset
|
||||
X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)
|
||||
# define models and parameters
|
||||
model = RidgeClassifier()
|
||||
alpha = [0.9, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.1, 1.0]
|
||||
# define grid search
|
||||
grid = dict(alpha=alpha)
|
||||
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
|
||||
grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0)
|
||||
grid_result = grid_search.fit(X, y)
|
||||
# summarize results
|
||||
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
|
||||
means = grid_result.cv_results_['mean_test_score']
|
||||
stds = grid_result.cv_results_['std_test_score']
|
||||
params = grid_result.cv_results_['params']
|
||||
for mean, stdev, param in zip(means, stds, params):
|
||||
print("%f (%f) with: %r" % (mean, stdev, param))
|
||||
|
||||
# NOTES:
|
||||
# alpha: If all alphas return the same mean, which do you chose?
|
||||
# Python seems to chose the first one?
|
||||
# https://stats.stackexchange.com/questions/166950/alpha-parameter-in-ridge-regression-is-high
|
||||
# The L2 norm term in ridge regression is weighted by the regularization parameter
|
||||
# alpha. So, the alpha parameter need not be small. But, for a larger alpha, the
|
||||
# flexibility of the fit would be very strict.
|
||||
|
||||
#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn
|
||||
# If you only want accuracy, then you can simply use cross_val_score()
|
||||
# kf = KFold(n_splits=10)
|
||||
# clf_tree=DecisionTreeClassifier()
|
||||
# scores = cross_val_score(clf_tree, X, y, cv=kf)
|
||||
# avg_score = np.mean(score_array)
|
||||
# print(avg_score)
|
||||
# Here cross_val_score will take as input your original X and y (without splitting into train and test). cross_val_score will automatically split them into train and test, fit the model on train data and score on test data. And those scores will be returned in the scores variable.
|
||||
# So when you have 10 folds, 10 scores will be returned in scores variable. You can then just take an average of that.
|
||||
# So, if the alpha value is 0, it means that it is just an Ordinary Least Squares
|
||||
# Regression model. So, the larger is the alpha, the higher is the smoothness constraint.
|
||||
# So, the smaller the value of alpha, the higher would be the magnitude of the coefficients.
|
||||
|
||||
mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
|
||||
# Could be that the model does not fit very well. With a very large alpha,
|
||||
# the algorithm more or else ignores the IV's and fits a mean. – Placidia
|
||||
# @Placidia, yes I would completely agree with your comment. I was just trying to
|
||||
# explain the significance of alpha as a parameter (as asked in the question) in
|
||||
# Ridge Regression, and how it's change would affect the fit and the coefficients.
|
||||
# Thank you for including the point in the comment.
|
||||
# ** READ: https://machinelearningcompass.com/machine_learning_models/ridge_regression/
|
||||
#%% KNeighborsClassifier
|
||||
from sklearn.datasets import make_blobs
|
||||
from sklearn.model_selection import RepeatedStratifiedKFold
|
||||
from sklearn.model_selection import GridSearchCV
|
||||
from sklearn.neighbors import KNeighborsClassifier
|
||||
# define dataset
|
||||
X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)
|
||||
# define models and parameters
|
||||
model = KNeighborsClassifier()
|
||||
n_neighbors = range(1, 21, 2)
|
||||
weights = ['uniform', 'distance']
|
||||
metric = ['euclidean', 'manhattan', 'minkowski']
|
||||
#p = [1,2]
|
||||
# define grid search
|
||||
grid = dict(n_neighbors=n_neighbors,weights=weights,metric=metric)
|
||||
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
|
||||
grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0)
|
||||
grid_result = grid_search.fit(X, y)
|
||||
# summarize results
|
||||
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
|
||||
means = grid_result.cv_results_['mean_test_score']
|
||||
stds = grid_result.cv_results_['std_test_score']
|
||||
params = grid_result.cv_results_['params']
|
||||
for mean, stdev, param in zip(means, stds, params):
|
||||
print("%f (%f) with: %r" % (mean, stdev, param))
|
||||
|
||||
# NOTES:
|
||||
# https://vitalflux.com/k-nearest-neighbors-explained-with-python-examples/
|
||||
# https://vitalflux.com/overfitting-underfitting-concepts-interview-questions/
|
||||
# Larger value of K ==> model may underfit
|
||||
# Smaller value of K ==> the model may overfit.
|
||||
#%%Support Vector Machine (SVM)
|
||||
# example of grid searching key hyperparametres for SVC
|
||||
from sklearn.datasets import make_blobs
|
||||
from sklearn.model_selection import RepeatedStratifiedKFold
|
||||
from sklearn.model_selection import GridSearchCV
|
||||
from sklearn.svm import SVC
|
||||
# define dataset
|
||||
X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)
|
||||
# define model and parameters
|
||||
model = SVC()
|
||||
kernel = ['poly', 'rbf', 'sigmoid']
|
||||
C = [50, 10, 1.0, 0.1, 0.01]
|
||||
gamma = ['scale']
|
||||
# define grid search
|
||||
grid = dict(kernel=kernel,C=C,gamma=gamma)
|
||||
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
|
||||
grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0)
|
||||
grid_result = grid_search.fit(X, y)
|
||||
# summarize results
|
||||
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
|
||||
means = grid_result.cv_results_['mean_test_score']
|
||||
stds = grid_result.cv_results_['std_test_score']
|
||||
params = grid_result.cv_results_['params']
|
||||
for mean, stdev, param in zip(means, stds, params):
|
||||
print("%f (%f) with: %r" % (mean, stdev, param))
|
||||
|
||||
# NOTES:
|
||||
# https://stats.stackexchange.com/questions/31066/what-is-the-influence-of-c-in-svms-with-linear-kernel
|
||||
# SVM terms: hyperplane, C and soft margins
|
||||
# hyperplane that can min(max(dist)) of the suppor vectors from tne hyperplane
|
||||
# High C ==> increase overfitting
|
||||
# Low C ==> increase underfitting
|
||||
|
||||
scoring_refit_recall = {'scoring': 'recall'
|
||||
,'refit': 'recall'}
|
||||
# But if C is a regularization parameter, why does a high C increase
|
||||
# overfitting, when generally speaking regularization is done to
|
||||
# mitigate overfitting, i.e., by creating a more general model?
|
||||
# C is a regularisation parameter, but it is essentially attached to
|
||||
# the data misfit term (the sum of the slack variables) rather than
|
||||
# the regularisation term (the margin bit), so a larger value of C
|
||||
# means less regularisation, rather than more. Alternatively you can
|
||||
# view the usual representation of the rgularisation parameter
|
||||
# as 1/C.
|
||||
|
||||
scoring_refit_recall = {'scoring': 'precision'
|
||||
,'refit': 'precision'}
|
||||
|
||||
scoring_refit_mcc = {'scoring': mcc_score_fn
|
||||
,'refit': 'mcc'}
|
||||
#n_jobs = 10 # my desktop has 12 cores
|
||||
#cv = {'cv': 10}#%%
|
||||
|
||||
njobs = {'n_jobs': 10}
|
||||
skf_cv = StratifiedKFold(n_splits = 10, shuffle = True)
|
||||
|
||||
#%% GSCV: RandomForest
|
||||
gs_rf = GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, oob_score = True
|
||||
#,class_weight = {1: 10/11, 0: 1/11}
|
||||
)
|
||||
, param_grid=[{'max_depth': [4, 6, 8, 10, None]
|
||||
, 'max_features': ['auto', 'sqrt']
|
||||
, 'min_samples_leaf': [2, 4, 8]
|
||||
, 'min_samples_split': [10, 20]}]
|
||||
, cv = skf_cv
|
||||
, **njobs
|
||||
, **scoring_refit_recall
|
||||
#, **scoring_refit_mcc
|
||||
#, scoring = scoring_fn, refit = False
|
||||
)
|
||||
#gs_rf.fit(X_train, y_train)
|
||||
#gs_rf_fit = gs_rf.fit(X_train y_train)
|
||||
|
||||
gs_rf.fit(X, y)
|
||||
gs_rf_fit = gs_rf.fit(X, y)
|
||||
gs_rf_res = gs_rf_fit.cv_results_
|
||||
print('Best model:\n', gs_rf.best_params_)
|
||||
print('Best models score:\n', gs_rf.best_score_)
|
||||
print('Check mean models score:\n', mean(gs_rf_res['mean_test_score']))
|
||||
|
||||
#%% Proof of concept: manual inspection to see how best score is calcualted!
|
||||
# SATISFIED!
|
||||
# Best_model example: recall, Best model's score: 0.8059288537549408
|
||||
# {'max_depth': 4, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 10}
|
||||
|
||||
# Best model example: mcc, Best models score: 0.42504894661702863
|
||||
# {'max_depth': 4, 'max_features': 'auto', 'min_samples_leaf': 4, 'min_samples_split': 20}
|
||||
|
||||
# Best model example: precision, Best models score: 0.7144745254745255
|
||||
# {'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10}
|
||||
|
||||
best_model = [{'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10 }]
|
||||
|
||||
gs_results_df = pd.DataFrame(gs_rf_res)
|
||||
gs_results_df.shape
|
||||
|
||||
gs_best_df = gs_results_df.loc[gs_results_df['params'].isin(best_model)]
|
||||
gs_best_df.shape
|
||||
|
||||
gs_best_df_test = gs_best_df.filter(like = 'test_', axis = 1)
|
||||
gs_best_df_test.shape
|
||||
|
||||
gs_best_df_test_recall = gs_best_df_test.filter(like = '_score', axis = 1)
|
||||
gs_best_df_test_recall.shape
|
||||
|
||||
f = gs_best_df_test_recall.filter(like='split', axis = 1)
|
||||
f.shape
|
||||
#gs_best_df_test_mcc = gs_best_df_test.filter(like = '_mcc', axis = 1)
|
||||
#f = gs_best_df_test_mcc.filter(like='split', axis = 1)
|
||||
|
||||
f.iloc[:,[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]].mean(axis = 1)
|
||||
# recall: 0.801186 vs 0.8059288537549408
|
||||
# mcc: 0.425049 vs 0.42504894661702863
|
||||
# precision: 0.714475 vs 0.7144745254745255
|
||||
|
||||
#%%
|
||||
|
||||
#%% Check the scores:
|
||||
print([(len(train), len(test)) for train, test in skf_cv.split(X, y)])
|
||||
gs_rf_fit.cv_results_
|
||||
#its the weighted average!?
|
||||
#%%
|
||||
#C is a regularization parameter that controls the trade off
|
||||
#between the achieving a low training error and a low testing
|
||||
# error that is the ability to generalize your classifier to unseen data.
|
||||
|
||||
# C Parameter is used for controlling the outliers:
|
||||
# low C implies ==> we are allowing more outliers
|
||||
# high C implies we are allowing fewer outliers.
|
||||
|
||||
|
|
|
|||
|
|
@ -81,6 +81,11 @@ njobs = {'n_jobs': 10}
|
|||
skf_cv = StratifiedKFold(n_splits = 10
|
||||
#, shuffle = False, random_state= None)
|
||||
, shuffle = True,**rs)
|
||||
rskf_cv = RepeatedStratifiedKFold(n_splits = 10
|
||||
, n_repeats=3
|
||||
#, shuffle = False, random_state= None)
|
||||
#, shuffle = True
|
||||
,**rs)
|
||||
#my_mcc = make_scorer({'mcc':make_scorer(matthews_corrcoef})
|
||||
mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ class ClfSwitcher(BaseEstimator):
|
|||
#def recall_score(self, X, y):
|
||||
# return self.estimator.recall_score(X, y)
|
||||
#%% Custom GridSearch: IntraModel[orig]
|
||||
def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical','mixed']) :
|
||||
def grid_search(input_df, target, sel_cv, var_type = ['numerical', 'categorical','mixed']) :
|
||||
|
||||
pipeline1 = Pipeline((
|
||||
('pre', MinMaxScaler())
|
||||
|
|
@ -73,7 +73,7 @@ def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical
|
|||
for i in range(len(pars)):
|
||||
print('IIIII===>', i)
|
||||
gs = GridSearchCV(pips[i], pars[i]
|
||||
, cv = skf_cv
|
||||
, cv = sel_cv
|
||||
, **scoring_refit
|
||||
#, refit=False
|
||||
, **njobs
|
||||
|
|
@ -82,9 +82,21 @@ def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical
|
|||
print ("finished Gridsearch")
|
||||
print ('\nBest model:', gs.best_params_)
|
||||
print ('\nBest score:', gs.best_score_)
|
||||
#%% Custom grid_search: Intra-Model [with return]
|
||||
# TODO: add
|
||||
# # summarize results
|
||||
# print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
|
||||
# means = grid_result.cv_results_['mean_test_score']
|
||||
# stds = grid_result.cv_results_['std_test_score']
|
||||
# params = grid_result.cv_results_['params']
|
||||
# for mean, stdev, param in zip(means, stds, params):
|
||||
# print("%f (%f) with: %r" % (mean, stdev, param))
|
||||
|
||||
# CALL: grid_search [orig]
|
||||
grid_search()
|
||||
|
||||
# #%% Custom grid_search: Intra-Model [with return]
|
||||
def grid_search(input_df, target
|
||||
, skf_cv
|
||||
, sel_cv
|
||||
, chosen_scoreD #scoring_refit
|
||||
#, var_type = ['numerical', 'categorical','mixed']
|
||||
):
|
||||
|
|
@ -128,7 +140,7 @@ def grid_search(input_df, target
|
|||
|
||||
print("\nStarting Gridsearch for model:", model_name, i)
|
||||
gs = GridSearchCV(all_pipelines[i], all_parameters[i]
|
||||
, cv = skf_cv
|
||||
, cv = sel_cv
|
||||
#, **scoring_refit
|
||||
#, refit=False
|
||||
, **chosen_scoreD
|
||||
|
|
@ -150,6 +162,9 @@ def grid_search(input_df, target
|
|||
out[model_name].update(chosen_scoreD.copy())
|
||||
out[model_name].update({'best_score': gs.best_score_}.copy())
|
||||
return(out)
|
||||
|
||||
# TODO:
|
||||
# print, or see for each model mean test score and sd, sometimes they can be identical and your best model just picks one!
|
||||
#%% call CUSTOM grid_search: INTRA model [with return]
|
||||
# call
|
||||
chosen_score = {'scoring': 'recall'
|
||||
|
|
@ -158,7 +173,6 @@ mcc_score_fn = {'chosen_scoreD': {'scoring': {'mcc': make_scorer(matthews_corrco
|
|||
,'refit': 'mcc'}
|
||||
}
|
||||
}
|
||||
|
||||
intra_models = grid_search(X, y
|
||||
, skf_cv = skf_cv
|
||||
, chosen_scoreD= chosen_score
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ njobs = {'n_jobs': 10}
|
|||
# TODO: get accuracy and other scores through K-fold cv
|
||||
|
||||
# Multiple Classification - Model Pipeline
|
||||
def MultClassPipeSKFLoop(input_df, target, skf_cv, var_type = ['numerical','categorical','mixed']):
|
||||
def MultClassPipeSKFLoop(input_df, target, sel_cv, var_type = ['numerical','categorical','mixed']):
|
||||
|
||||
'''
|
||||
@ param input_df: input features
|
||||
|
|
@ -131,7 +131,7 @@ def MultClassPipeSKFLoop(input_df, target, skf_cv, var_type = ['numerical','cate
|
|||
fold_dict.update({ model_name: {}})
|
||||
|
||||
#scores_df = pd.DataFrame()
|
||||
for train_index, test_index in skf_cv.split(input_df, target):
|
||||
for train_index, test_index in sel_cv.split(input_df, target):
|
||||
x_train_fold, x_test_fold = input_df.iloc[train_index], input_df.iloc[test_index]
|
||||
y_train_fold, y_test_fold = target.iloc[train_index], target.iloc[test_index]
|
||||
#print("Fold: ", fold_no, len(train_index), len(test_index))
|
||||
|
|
|
|||
|
|
@ -6,16 +6,17 @@ Created on Fri Mar 11 11:15:50 2022
|
|||
@author: tanu
|
||||
"""
|
||||
#%% variables
|
||||
rs = {'random_state': 42}
|
||||
# rs = {'random_state': 42}
|
||||
|
||||
skf_cv = StratifiedKFold(n_splits = 10
|
||||
#, shuffle = False, random_state= None)
|
||||
, shuffle = True,**rs)
|
||||
# skf_cv = StratifiedKFold(n_splits = 10
|
||||
# #, shuffle = False, random_state= None)
|
||||
# , shuffle = True,**rs)
|
||||
#%% MultClassPipeSKFLoop: function call()
|
||||
t3_res = MultClassPipeSKFLoop(input_df = num_df_wtgt[numerical_FN]
|
||||
, target = num_df_wtgt['mutation_class']
|
||||
, var_type = 'numerical'
|
||||
, skf_cv = skf_cv)
|
||||
, sel_cv = skf_cv)
|
||||
#, sel_cv = rskf_cv)
|
||||
pp.pprint(t3_res)
|
||||
#print(t3_res)
|
||||
################################################################
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue