diff --git a/__pycache__/loopity_loop.cpython-37.pyc b/__pycache__/loopity_loop.cpython-37.pyc index f1efd32..994b92f 100644 Binary files a/__pycache__/loopity_loop.cpython-37.pyc and b/__pycache__/loopity_loop.cpython-37.pyc differ diff --git a/classification_algo_names.py b/classification_algo_names.py index 46f4822..905c152 100644 --- a/classification_algo_names.py +++ b/classification_algo_names.py @@ -15,6 +15,8 @@ Created on Fri Mar 18 09:47:48 2022 # https://uk.mathworks.com/help/stats/hyperparameter-optimization-in-classification-learner-app.html [ algo] # As a general rule of thumb, it is required to run baseline models on the dataset. I know H2O- AutoML and other AutoML packages do this. But I want to try using Scikit-learn Pipeline, # https://codereview.stackexchange.com/questions/256934/model-pipeline-to-run-multiple-classifiers-for-ml-classification +# https://uk.mathworks.com/help/stats/hyperparameter-optimization-in-classification-learner-app.html +# QDA: https://www.geeksforgeeks.org/quadratic-discriminant-analysis/ names = [ "Nearest Neighbors", @@ -41,3 +43,222 @@ classifiers = [ GaussianNB(), QuadraticDiscriminantAnalysis(), ] + +# NOTE Logistic regression +# The choice of the algorithm depends on the penalty chosen: Supported penalties by solver: +# ‘newton-cg’ - [‘l2’, ‘none’] +# ‘lbfgs’ - [‘l2’, ‘none’] +# ‘liblinear’ - [‘l1’, ‘l2’] +# ‘sag’ - [‘l2’, ‘none’] +# ‘saga’ - [‘elasticnet’, ‘l1’, ‘l2’, ‘none’] + +# SVR? +# estimator=SVR(kernel='rbf') +# param_grid={ +# 'C': [1.1, 5.4, 170, 1001], +# 'epsilon': [0.0003, 0.007, 0.0109, 0.019, 0.14, 0.05, 8, 0.2, 3, 2, 7], +# 'gamma': [0.7001, 0.008, 0.001, 3.1, 1, 1.3, 5] +# } + + +#%% Classification algorithms param grid +#%% LogisticRegression() +#https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html + gs_lr = Pipeline(( + ('pre' , MinMaxScaler()) + ,('clf', LogisticRegression(**rs + , **njobs)) + )) + gs_lr_params = { + 'clf__C' : [0.0001, 0.001, 0.01, 0.1 ,1, 10, 100] + #'C': np.logspace(-4, 4, 50) + , 'clf__penalty': ['l1', 'l2', 'elasticnet', 'none'] + , 'clf__solver' : ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'] + } +#%% DecisionTreeClassifier() + + gs_dt = Pipeline(( + ('pre' , MinMaxScaler()) + , ('clf', DecisionTreeClassifier(**rs + , **njobs)) + )) + gs_dt_params = { + 'clf__max_depth': [ 2, 4, 6, 8, 10] + , 'clf__criterion':['gini','entropy'] + , "clf__max_features":["auto", None] + , "clf__max_leaf_nodes":[10,20,30,40] + } +#%% KNeighborsClassifier() +#https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html + gs_knn = Pipeline(( + ('pre' , MinMaxScaler()) + ,('clf', KNeighborsClassifier(**rs + , **njobs)) + )) + + gs_knn_params = { + 'clf__n_neighbors': [3, 7, 10] + #, 'clf__n_neighbors': range(1, 21, 2) + ,'clf__metric' : ['euclidean', 'manhattan', 'minkowski'] + , 'clf__weights' : ['uniform', 'distance'] + } +#%% RandomForestClassifier() + + gs_rf = Pipeline(( + ('pre' , MinMaxScaler()) + ,('clf', RandomForestClassifier(**rs + , **njobs + , bootstrap = True + , oob_score = True)) + )) + gs_rf_params = { + 'clf__max_depth': [4, 6, 8, 10, 12, 16, 20, None] + , 'clf__class_weight':['balanced','balanced_subsample'] + , 'clf__n_estimators': [10, 100, 1000] + , 'clf__criterion': ['gini', 'entropy'] + , 'clf__max_features': ['auto', 'sqrt'] + , 'clf__min_samples_leaf': [2, 4, 8, 50] + , 'clf__min_samples_split': [10, 20] + } +#%% XGBClassifier() +# https://stackoverflow.com/questions/34674797/xgboost-xgbclassifier-defaults-in-python +# https://stackoverflow.com/questions/34674797/xgboost-xgbclassifier-defaults-in-python + gs_xgb = Pipeline(( + ('pre' , MinMaxScaler()) + ,('clf', XGBClassifier(**rs + , **njobs)) + )) + + gs_xgb_params = { + 'clf__learning_rate': [0.01, 0.05, 0.1, 0.2] + , 'clf__max_depth': [4, 6, 8, 10, 12, 16, 20] + , 'clf__min_samples_leaf': [4, 8, 12, 16, 20] + , 'clf__max_features': ['auto', 'sqrt'] + } +#%% MLPClassifier() +# https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html + gs_mlp = Pipeline(( + ('pre' , MinMaxScaler()) + ,('clf', MLPClassifier(**rs + , **njobs + , max_iter = 500)) + )) + + gs_mlp_params = { + 'clf__hidden_layer_sizes': [(1), (2), (3)] + , 'clf__max_features': ['auto', 'sqrt'] + , 'clf__min_samples_leaf': [2, 4, 8] + , 'clf__min_samples_split': [10, 20] + } +#%% RidgeClassifier() +# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.RidgeClassifier.html + gs_rc = Pipeline(( + ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1 + ,('clf', RidgeClassifier(**rs + , **njobs)) + )) + + gs_rc_params = { + 'clf__alpha': [0.1, 0.2, 0.5, 0.8, 1.0] + } + +#%% SVC() +# https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html + gs_svc = Pipeline(( + ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1 + ,('clf', SVC(**rs + , **njobs)) + )) + + gs_svc_params = { + 'clf__kernel': ['linear', 'poly', 'rbf', 'sigmoid', 'precomputed'} + , 'clf__C' : [50, 10, 1.0, 0.1, 0.01] + , 'clf__gamma': ['scale', 'auto'] } + +#%% BaggingClassifier() +#https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.BaggingClassifier.html + gs_bdt = Pipeline(( + ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1 + ,('clf', BaggingClassifier(**rs + , **njobs + , bootstrap = True + , oob_score = True)) + )) + + gs_bdt_params = { + 'clf__n_estimators' : [10, 100, 1000] + # If None, then the base estimator is a DecisionTreeClassifier. + , 'clf__base_estimator' : ['None', 'SVC()', 'KNeighborsClassifier()']# if none, DT is used + , 'clf__gamma': ['scale', 'auto'] } +#%% GradientBoostingClassifier() +# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html + gs_gb = Pipeline(( + ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1 + ,('clf', GradientBoostingClassifier(**rs)) + )) + + gs_bdt_params = { + 'clf__n_estimators' : [10, 100, 1000] + , 'clf__n_estimators' : [10, 100, 1000] + , 'clf__learning_rate': [0.001, 0.01, 0.1] + , 'clf__subsample' : [0.5, 0.7, 1.0] + , 'clf__max_depth' : [3, 7, 9] + } +#%% AdaBoostClassifier() +#https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html#sklearn.ensemble.AdaBoostClassifier + gs_gb = Pipeline(( + ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1 + ,('clf', AdaBoostClassifier(**rs)) + )) + + gs_bdt_params = { + 'clf__n_estimators': [none, 1, 2] + , 'clf__base_estiamtor' : ['None', 1*SVC(), 1*KNeighborsClassifier()] + #, 'clf___splitter' : ["best", "random"] + } +#%% GaussianProcessClassifier() +# https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.GaussianProcessClassifier.html + #GaussianProcessClassifier(1.0 * RBF(1.0)), + gs_gpc = Pipeline(( + ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1 + ,('clf', GaussianProcessClassifier(**rs)) + )) + + gs_gpc_params = { + 'clf__kernel': [1*RBF(), 1*DotProduct(), 1*Matern(), 1*RationalQuadratic(), 1*WhiteKernel()] + } + +#%% GaussianNB() +# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html + gs_gnb = Pipeline(( + ('pre' , MinMaxScaler()) + , ('pca', PCA() )# CHECK if it wants -1 to 1 + ,('clf', GaussianNB(**rs)) + )) + + gs_gnb_params = { + 'clf__priors': [None] + , 'clf__var_smoothing': np.logspace(0,-9, num=100) + } + +#%% QuadraticDiscriminantAnalysis() +#https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html + + gs_qda = Pipeline(( + ('pre' , MinMaxScaler()) + #, ('pca', PCA() )# CHECK if it wants -1 to 1 + ,('clf', QuadraticDiscriminantAnalysis()) + )) +#%% BernoulliNB() +# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.BernoulliNB.html + gs_gnb = Pipeline(( + ('pre' , MinMaxScaler()) + ,('clf', BernoulliNB()) + )) +BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True) + gs_gnb_params = { + 'clf__alpha': [0, 1] + , 'clf__binarize':['None', 0] + , 'clf__fit_prior': [True] + , 'clf__class_prior': ['None'] + } \ No newline at end of file diff --git a/hyperparams.py b/hyperparams.py index e51324d..546ab6f 100644 --- a/hyperparams.py +++ b/hyperparams.py @@ -16,10 +16,9 @@ scoring_refit = {'scoring': mcc_score_fn ,'refit': 'mcc'} #n_jobs = 10 # my desktop has 12 cores njobs = {'n_jobs': 10} -skf_cv = StratifiedKFold(n_splits=10,shuffle = True) +skf_cv = StratifiedKFold(n_splits = 10, shuffle = True) #cv = {'cv': 10} - gs_dt = GridSearchCV(estimator=DecisionTreeClassifier(**rs #,class_weight = {1:10, 0:1} ), @@ -43,8 +42,8 @@ gs_dt_fit = gs_dt.fit(num_df_wtgt[numerical_FN] gs_dt_fit_res = gs_dt_fit.cv_results_ -print('Best model:\n', gs_dt.best_params_) -print('Best models score:\n', gs_dt.best_score_) +print('Best model:\n' , gs_dt.best_params_) +print('Best models score:\n' , gs_dt.best_score_) print('Check best models score:\n', mean(gs_dt_fit_res['mean_test_mcc'])) #%% Check the scores: @@ -106,3 +105,104 @@ means = clf.cv_results_['mean_test_score'] stds = clf.cv_results_['std_test_score'] for mean, std, params in zip(means, stds, clf.cv_results_['params']): print("%0.3f (+/-%0.03f) for %r" % (mean, std * 2, params)) + +######################################################################## +#%%: Hyperparams with SKF and trying different scoring functions + # https://stackoverflow.com/questions/57248072/gridsearchcv-gives-different-result +# https://stackoverflow.com/questions/44947574/what-is-the-meaning-of-mean-test-score-in-cv-result +#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn + + +#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn +# If you only want accuracy, then you can simply use cross_val_score() +# kf = KFold(n_splits=10) +# clf_tree=DecisionTreeClassifier() +# scores = cross_val_score(clf_tree, X, y, cv=kf) +# avg_score = np.mean(score_array) +# print(avg_score) +# Here cross_val_score will take as input your original X and y (without splitting into train and test). cross_val_score will automatically split them into train and test, fit the model on train data and score on test data. And those scores will be returned in the scores variable. +# So when you have 10 folds, 10 scores will be returned in scores variable. You can then just take an average of that. + +mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} + +scoring_refit_recall = {'scoring': 'recall' + ,'refit': 'recall'} + +scoring_refit_recall = {'scoring': 'precision' + ,'refit': 'precision'} + +scoring_refit_mcc = {'scoring': mcc_score_fn + ,'refit': 'mcc'} +#n_jobs = 10 # my desktop has 12 cores +#cv = {'cv': 10}#%% + +njobs = {'n_jobs': 10} +skf_cv = StratifiedKFold(n_splits = 10, shuffle = True) + +#%% GSCV: RandomForest +gs_rf = GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, oob_score = True + #,class_weight = {1: 10/11, 0: 1/11} + ) + , param_grid=[{'max_depth': [4, 6, 8, 10, None] + , 'max_features': ['auto', 'sqrt'] + , 'min_samples_leaf': [2, 4, 8] + , 'min_samples_split': [10, 20]}] + , cv = skf_cv + , **njobs + , **scoring_refit_recall + #, **scoring_refit_mcc + #, scoring = scoring_fn, refit = False + ) +#gs_rf.fit(X_train, y_train) +#gs_rf_fit = gs_rf.fit(X_train y_train) + +gs_rf.fit(X, y) +gs_rf_fit = gs_rf.fit(X, y) +gs_rf_res = gs_rf_fit.cv_results_ +print('Best model:\n', gs_rf.best_params_) +print('Best models score:\n', gs_rf.best_score_) +print('Check mean models score:\n', mean(gs_rf_res['mean_test_score'])) + +#%% Proof of concept: manual inspection to see how best score is calcualted! +# SATISFIED! +# Best_model example: recall, Best model's score: 0.8059288537549408 +# {'max_depth': 4, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 10} + +# Best model example: mcc, Best models score: 0.42504894661702863 +# {'max_depth': 4, 'max_features': 'auto', 'min_samples_leaf': 4, 'min_samples_split': 20} + +# Best model example: precision, Best models score: 0.7144745254745255 +# {'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10} + +best_model = [{'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10 }] + +gs_results_df = pd.DataFrame(gs_rf_res) +gs_results_df.shape + +gs_best_df = gs_results_df.loc[gs_results_df['params'].isin(best_model)] +gs_best_df.shape + +gs_best_df_test = gs_best_df.filter(like = 'test_', axis = 1) +gs_best_df_test.shape + +gs_best_df_test_recall = gs_best_df_test.filter(like = '_score', axis = 1) +gs_best_df_test_recall.shape + +f = gs_best_df_test_recall.filter(like='split', axis = 1) +f.shape +#gs_best_df_test_mcc = gs_best_df_test.filter(like = '_mcc', axis = 1) +#f = gs_best_df_test_mcc.filter(like='split', axis = 1) + +f.iloc[:,[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]].mean(axis = 1) +# recall: 0.801186 vs 0.8059288537549408 +# mcc: 0.425049 vs 0.42504894661702863 +# precision: 0.714475 vs 0.7144745254745255 + +#%% + +#%% Check the scores: +print([(len(train), len(test)) for train, test in skf_cv.split(X, y)]) +gs_rf_fit.cv_results_ +#its the weighted average!? +#%% + diff --git a/hyperparams_p1.py b/hyperparams_p1.py index 4b29506..b4dfdb7 100644 --- a/hyperparams_p1.py +++ b/hyperparams_p1.py @@ -1,106 +1,158 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -Created on Wed Mar 16 16:55:06 2022 +Created on Sun Mar 20 13:02:54 2022 @author: tanu """ -# https://stackoverflow.com/questions/57248072/gridsearchcv-gives-different-result -# https://stackoverflow.com/questions/44947574/what-is-the-meaning-of-mean-test-score-in-cv-result -#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn +# https://machinelearningmastery.com/hyperparameters-for-classification-machine-learning-algorithms/ +#%% LogisticRegression +# example of grid searching key hyperparametres for logistic regression +from sklearn.datasets import make_blobs +from sklearn.model_selection import RepeatedStratifiedKFold +from sklearn.model_selection import GridSearchCV +from sklearn.linear_model import LogisticRegression +# define dataset +X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20) +# define models and parameters +model = LogisticRegression() +solvers = ['newton-cg', 'lbfgs', 'liblinear'] +penalty = ['l2'] +c_values = [100, 10, 1.0, 0.1, 0.01] +# define grid search +grid = dict(solver=solvers,penalty=penalty,C=c_values) +cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1) +grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0) +grid_result = grid_search.fit(X, y) +# summarize results +print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) +means = grid_result.cv_results_['mean_test_score'] +stds = grid_result.cv_results_['std_test_score'] +params = grid_result.cv_results_['params'] +for mean, stdev, param in zip(means, stds, params): + print("%f (%f) with: %r" % (mean, stdev, param)) +#%% RidgeClassifier +from sklearn.datasets import make_blobs +from sklearn.model_selection import RepeatedStratifiedKFold +from sklearn.model_selection import GridSearchCV +from sklearn.linear_model import RidgeClassifier +# define dataset +X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20) +# define models and parameters +model = RidgeClassifier() +alpha = [0.9, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.1, 1.0] +# define grid search +grid = dict(alpha=alpha) +cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1) +grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0) +grid_result = grid_search.fit(X, y) +# summarize results +print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) +means = grid_result.cv_results_['mean_test_score'] +stds = grid_result.cv_results_['std_test_score'] +params = grid_result.cv_results_['params'] +for mean, stdev, param in zip(means, stds, params): + print("%f (%f) with: %r" % (mean, stdev, param)) + +# NOTES: +# alpha: If all alphas return the same mean, which do you chose? +# Python seems to chose the first one? +# https://stats.stackexchange.com/questions/166950/alpha-parameter-in-ridge-regression-is-high +# The L2 norm term in ridge regression is weighted by the regularization parameter +# alpha. So, the alpha parameter need not be small. But, for a larger alpha, the +# flexibility of the fit would be very strict. -#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn -# If you only want accuracy, then you can simply use cross_val_score() -# kf = KFold(n_splits=10) -# clf_tree=DecisionTreeClassifier() -# scores = cross_val_score(clf_tree, X, y, cv=kf) -# avg_score = np.mean(score_array) -# print(avg_score) -# Here cross_val_score will take as input your original X and y (without splitting into train and test). cross_val_score will automatically split them into train and test, fit the model on train data and score on test data. And those scores will be returned in the scores variable. -# So when you have 10 folds, 10 scores will be returned in scores variable. You can then just take an average of that. +# So, if the alpha value is 0, it means that it is just an Ordinary Least Squares +# Regression model. So, the larger is the alpha, the higher is the smoothness constraint. +# So, the smaller the value of alpha, the higher would be the magnitude of the coefficients. -mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} +# Could be that the model does not fit very well. With a very large alpha, +# the algorithm more or else ignores the IV's and fits a mean. – Placidia +# @Placidia, yes I would completely agree with your comment. I was just trying to +# explain the significance of alpha as a parameter (as asked in the question) in +# Ridge Regression, and how it's change would affect the fit and the coefficients. +# Thank you for including the point in the comment. +# ** READ: https://machinelearningcompass.com/machine_learning_models/ridge_regression/ +#%% KNeighborsClassifier +from sklearn.datasets import make_blobs +from sklearn.model_selection import RepeatedStratifiedKFold +from sklearn.model_selection import GridSearchCV +from sklearn.neighbors import KNeighborsClassifier +# define dataset +X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20) +# define models and parameters +model = KNeighborsClassifier() +n_neighbors = range(1, 21, 2) +weights = ['uniform', 'distance'] +metric = ['euclidean', 'manhattan', 'minkowski'] +#p = [1,2] +# define grid search +grid = dict(n_neighbors=n_neighbors,weights=weights,metric=metric) +cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1) +grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0) +grid_result = grid_search.fit(X, y) +# summarize results +print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) +means = grid_result.cv_results_['mean_test_score'] +stds = grid_result.cv_results_['std_test_score'] +params = grid_result.cv_results_['params'] +for mean, stdev, param in zip(means, stds, params): + print("%f (%f) with: %r" % (mean, stdev, param)) + +# NOTES: +# https://vitalflux.com/k-nearest-neighbors-explained-with-python-examples/ +# https://vitalflux.com/overfitting-underfitting-concepts-interview-questions/ +# Larger value of K ==> model may underfit +# Smaller value of K ==> the model may overfit. +#%%Support Vector Machine (SVM) +# example of grid searching key hyperparametres for SVC +from sklearn.datasets import make_blobs +from sklearn.model_selection import RepeatedStratifiedKFold +from sklearn.model_selection import GridSearchCV +from sklearn.svm import SVC +# define dataset +X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20) +# define model and parameters +model = SVC() +kernel = ['poly', 'rbf', 'sigmoid'] +C = [50, 10, 1.0, 0.1, 0.01] +gamma = ['scale'] +# define grid search +grid = dict(kernel=kernel,C=C,gamma=gamma) +cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1) +grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0) +grid_result = grid_search.fit(X, y) +# summarize results +print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) +means = grid_result.cv_results_['mean_test_score'] +stds = grid_result.cv_results_['std_test_score'] +params = grid_result.cv_results_['params'] +for mean, stdev, param in zip(means, stds, params): + print("%f (%f) with: %r" % (mean, stdev, param)) + +# NOTES: +# https://stats.stackexchange.com/questions/31066/what-is-the-influence-of-c-in-svms-with-linear-kernel +# SVM terms: hyperplane, C and soft margins +# hyperplane that can min(max(dist)) of the suppor vectors from tne hyperplane +# High C ==> increase overfitting +# Low C ==> increase underfitting -scoring_refit_recall = {'scoring': 'recall' - ,'refit': 'recall'} +# But if C is a regularization parameter, why does a high C increase +# overfitting, when generally speaking regularization is done to +# mitigate overfitting, i.e., by creating a more general model? +# C is a regularisation parameter, but it is essentially attached to +# the data misfit term (the sum of the slack variables) rather than +# the regularisation term (the margin bit), so a larger value of C +# means less regularisation, rather than more. Alternatively you can +# view the usual representation of the rgularisation parameter +# as 1/C. -scoring_refit_recall = {'scoring': 'precision' - ,'refit': 'precision'} - -scoring_refit_mcc = {'scoring': mcc_score_fn - ,'refit': 'mcc'} -#n_jobs = 10 # my desktop has 12 cores -#cv = {'cv': 10}#%% - -njobs = {'n_jobs': 10} -skf_cv = StratifiedKFold(n_splits = 10, shuffle = True) - -#%% GSCV: RandomForest -gs_rf = GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, oob_score = True - #,class_weight = {1: 10/11, 0: 1/11} - ) - , param_grid=[{'max_depth': [4, 6, 8, 10, None] - , 'max_features': ['auto', 'sqrt'] - , 'min_samples_leaf': [2, 4, 8] - , 'min_samples_split': [10, 20]}] - , cv = skf_cv - , **njobs - , **scoring_refit_recall - #, **scoring_refit_mcc - #, scoring = scoring_fn, refit = False - ) -#gs_rf.fit(X_train, y_train) -#gs_rf_fit = gs_rf.fit(X_train y_train) - -gs_rf.fit(X, y) -gs_rf_fit = gs_rf.fit(X, y) -gs_rf_res = gs_rf_fit.cv_results_ -print('Best model:\n', gs_rf.best_params_) -print('Best models score:\n', gs_rf.best_score_) -print('Check mean models score:\n', mean(gs_rf_res['mean_test_score'])) - -#%% Proof of concept: manual inspection to see how best score is calcualted! -# SATISFIED! -# Best_model example: recall, Best model's score: 0.8059288537549408 -# {'max_depth': 4, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 10} - -# Best model example: mcc, Best models score: 0.42504894661702863 -# {'max_depth': 4, 'max_features': 'auto', 'min_samples_leaf': 4, 'min_samples_split': 20} - -# Best model example: precision, Best models score: 0.7144745254745255 -# {'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10} - -best_model = [{'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10 }] - -gs_results_df = pd.DataFrame(gs_rf_res) -gs_results_df.shape - -gs_best_df = gs_results_df.loc[gs_results_df['params'].isin(best_model)] -gs_best_df.shape - -gs_best_df_test = gs_best_df.filter(like = 'test_', axis = 1) -gs_best_df_test.shape - -gs_best_df_test_recall = gs_best_df_test.filter(like = '_score', axis = 1) -gs_best_df_test_recall.shape - -f = gs_best_df_test_recall.filter(like='split', axis = 1) -f.shape -#gs_best_df_test_mcc = gs_best_df_test.filter(like = '_mcc', axis = 1) -#f = gs_best_df_test_mcc.filter(like='split', axis = 1) - -f.iloc[:,[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]].mean(axis = 1) -# recall: 0.801186 vs 0.8059288537549408 -# mcc: 0.425049 vs 0.42504894661702863 -# precision: 0.714475 vs 0.7144745254745255 - -#%% - -#%% Check the scores: -print([(len(train), len(test)) for train, test in skf_cv.split(X, y)]) -gs_rf_fit.cv_results_ -#its the weighted average!? -#%% +#C is a regularization parameter that controls the trade off +#between the achieving a low training error and a low testing +# error that is the ability to generalize your classifier to unseen data. +# C Parameter is used for controlling the outliers: +# low C implies ==> we are allowing more outliers +# high C implies we are allowing fewer outliers. diff --git a/imports.py b/imports.py index b5938a3..c6a2531 100644 --- a/imports.py +++ b/imports.py @@ -81,6 +81,11 @@ njobs = {'n_jobs': 10} skf_cv = StratifiedKFold(n_splits = 10 #, shuffle = False, random_state= None) , shuffle = True,**rs) +rskf_cv = RepeatedStratifiedKFold(n_splits = 10 + , n_repeats=3 + #, shuffle = False, random_state= None) + #, shuffle = True + ,**rs) #my_mcc = make_scorer({'mcc':make_scorer(matthews_corrcoef}) mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} diff --git a/intra_model_gscv.py b/intra_model_gscv.py index 40ef39e..a4f905c 100644 --- a/intra_model_gscv.py +++ b/intra_model_gscv.py @@ -37,7 +37,7 @@ class ClfSwitcher(BaseEstimator): #def recall_score(self, X, y): # return self.estimator.recall_score(X, y) #%% Custom GridSearch: IntraModel[orig] -def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical','mixed']) : +def grid_search(input_df, target, sel_cv, var_type = ['numerical', 'categorical','mixed']) : pipeline1 = Pipeline(( ('pre', MinMaxScaler()) @@ -73,7 +73,7 @@ def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical for i in range(len(pars)): print('IIIII===>', i) gs = GridSearchCV(pips[i], pars[i] - , cv = skf_cv + , cv = sel_cv , **scoring_refit #, refit=False , **njobs @@ -82,9 +82,21 @@ def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical print ("finished Gridsearch") print ('\nBest model:', gs.best_params_) print ('\nBest score:', gs.best_score_) -#%% Custom grid_search: Intra-Model [with return] +# TODO: add +# # summarize results +# print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) +# means = grid_result.cv_results_['mean_test_score'] +# stds = grid_result.cv_results_['std_test_score'] +# params = grid_result.cv_results_['params'] +# for mean, stdev, param in zip(means, stds, params): +# print("%f (%f) with: %r" % (mean, stdev, param)) + +# CALL: grid_search [orig] +grid_search() + +# #%% Custom grid_search: Intra-Model [with return] def grid_search(input_df, target - , skf_cv + , sel_cv , chosen_scoreD #scoring_refit #, var_type = ['numerical', 'categorical','mixed'] ): @@ -128,7 +140,7 @@ def grid_search(input_df, target print("\nStarting Gridsearch for model:", model_name, i) gs = GridSearchCV(all_pipelines[i], all_parameters[i] - , cv = skf_cv + , cv = sel_cv #, **scoring_refit #, refit=False , **chosen_scoreD @@ -150,6 +162,9 @@ def grid_search(input_df, target out[model_name].update(chosen_scoreD.copy()) out[model_name].update({'best_score': gs.best_score_}.copy()) return(out) + +# TODO: +# print, or see for each model mean test score and sd, sometimes they can be identical and your best model just picks one! #%% call CUSTOM grid_search: INTRA model [with return] # call chosen_score = {'scoring': 'recall' @@ -158,7 +173,6 @@ mcc_score_fn = {'chosen_scoreD': {'scoring': {'mcc': make_scorer(matthews_corrco ,'refit': 'mcc'} } } - intra_models = grid_search(X, y , skf_cv = skf_cv , chosen_scoreD= chosen_score diff --git a/loopity_loop.py b/loopity_loop.py index a0afc35..077ca66 100644 --- a/loopity_loop.py +++ b/loopity_loop.py @@ -40,7 +40,7 @@ njobs = {'n_jobs': 10} # TODO: get accuracy and other scores through K-fold cv # Multiple Classification - Model Pipeline -def MultClassPipeSKFLoop(input_df, target, skf_cv, var_type = ['numerical','categorical','mixed']): +def MultClassPipeSKFLoop(input_df, target, sel_cv, var_type = ['numerical','categorical','mixed']): ''' @ param input_df: input features @@ -131,7 +131,7 @@ def MultClassPipeSKFLoop(input_df, target, skf_cv, var_type = ['numerical','cate fold_dict.update({ model_name: {}}) #scores_df = pd.DataFrame() - for train_index, test_index in skf_cv.split(input_df, target): + for train_index, test_index in sel_cv.split(input_df, target): x_train_fold, x_test_fold = input_df.iloc[train_index], input_df.iloc[test_index] y_train_fold, y_test_fold = target.iloc[train_index], target.iloc[test_index] #print("Fold: ", fold_no, len(train_index), len(test_index)) diff --git a/loopity_loop_CALL.py b/loopity_loop_CALL.py index e70763e..e510a42 100644 --- a/loopity_loop_CALL.py +++ b/loopity_loop_CALL.py @@ -6,16 +6,17 @@ Created on Fri Mar 11 11:15:50 2022 @author: tanu """ #%% variables -rs = {'random_state': 42} +# rs = {'random_state': 42} -skf_cv = StratifiedKFold(n_splits = 10 - #, shuffle = False, random_state= None) - , shuffle = True,**rs) +# skf_cv = StratifiedKFold(n_splits = 10 +# #, shuffle = False, random_state= None) +# , shuffle = True,**rs) #%% MultClassPipeSKFLoop: function call() t3_res = MultClassPipeSKFLoop(input_df = num_df_wtgt[numerical_FN] , target = num_df_wtgt['mutation_class'] , var_type = 'numerical' - , skf_cv = skf_cv) + , sel_cv = skf_cv) + #, sel_cv = rskf_cv) pp.pprint(t3_res) #print(t3_res) ################################################################