From 0c4f1e1e5ffce8073dabfb54560e3a3b53d2ecaf Mon Sep 17 00:00:00 2001
From: Tanushree Tunstall <tanu@tunstall.in>
Date: Mon, 21 Mar 2022 13:51:20 +0000
Subject: [PATCH] added all classification algorithms params for gridsearch

---
 __pycache__/loopity_loop.cpython-37.pyc | Bin 4323 -> 4323 bytes
 classification_algo_names.py            | 221 ++++++++++++++++++++++
 hyperparams.py                          | 108 ++++++++++-
 hyperparams_p1.py                       | 238 +++++++++++++++---------
 imports.py                              |   5 +
 intra_model_gscv.py                     |  26 ++-
 loopity_loop.py                         |   4 +-
 loopity_loop_CALL.py                    |  11 +-
 8 files changed, 503 insertions(+), 110 deletions(-)

diff --git a/__pycache__/loopity_loop.cpython-37.pyc b/__pycache__/loopity_loop.cpython-37.pyc
index f1efd32f1fdfba334e5bcbbe9f04bfab4418c46e..994b92f60f0aecc06c7eb57c7f292c2cf036856e 100644
GIT binary patch
delta 93
zcmaE?_*jwKiI<m)0SH`l%{Ov~@~}p+6{qHGPUQ(@WUQLJoA(`K&E!>l<&3u`EAeXr
gNgsZ1#;D0N_!EHSPyTMv8lcvqDpaC+@=O6Q0M`y0e*gdg

delta 93
zcmaE?_*jwKiI<m)0SFRHj5l(J^03}wE6z^aoXQi*$XGpjH}5;f+R3Z<${C|3EAeXr
gNgsZ1##@tT@FxJtpZwjTwLq;!)u=?x<e36q01cxXW&i*H

diff --git a/classification_algo_names.py b/classification_algo_names.py
index 46f4822..905c152 100644
--- a/classification_algo_names.py
+++ b/classification_algo_names.py
@@ -15,6 +15,8 @@ Created on Fri Mar 18 09:47:48 2022
 # https://uk.mathworks.com/help/stats/hyperparameter-optimization-in-classification-learner-app.html [ algo]
 # As a general rule of thumb, it is required to run baseline models on the dataset. I know H2O- AutoML and other AutoML packages do this. But I want to try using Scikit-learn Pipeline,
     #  https://codereview.stackexchange.com/questions/256934/model-pipeline-to-run-multiple-classifiers-for-ml-classification
+# https://uk.mathworks.com/help/stats/hyperparameter-optimization-in-classification-learner-app.html
+# QDA: https://www.geeksforgeeks.org/quadratic-discriminant-analysis/
 
 names = [
     "Nearest Neighbors",
@@ -41,3 +43,222 @@ classifiers = [
     GaussianNB(),
     QuadraticDiscriminantAnalysis(),
 ]
+
+# NOTE Logistic regression
+# The choice of the algorithm depends on the penalty chosen: Supported penalties by solver:
+#     ‘newton-cg’ - [‘l2’, ‘none’]
+#     ‘lbfgs’ - [‘l2’, ‘none’]
+#     ‘liblinear’ - [‘l1’, ‘l2’]
+#     ‘sag’ - [‘l2’, ‘none’]
+#     ‘saga’ - [‘elasticnet’, ‘l1’, ‘l2’, ‘none’]
+
+# SVR?
+# estimator=SVR(kernel='rbf')
+# param_grid={
+#             'C': [1.1, 5.4, 170, 1001],
+#             'epsilon': [0.0003, 0.007, 0.0109, 0.019, 0.14, 0.05, 8, 0.2, 3, 2, 7],
+#             'gamma': [0.7001, 0.008, 0.001, 3.1, 1, 1.3, 5]
+#         }
+
+
+#%% Classification algorithms param grid
+#%% LogisticRegression()
+#https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
+    gs_lr = Pipeline((
+    ('pre' , MinMaxScaler())
+    ,('clf', LogisticRegression(**rs
+                                , **njobs))
+    ))
+    gs_lr_params = {
+        'clf__C'        : [0.0001, 0.001, 0.01, 0.1 ,1, 10, 100]
+        #'C': np.logspace(-4, 4, 50)
+        , 'clf__penalty': ['l1', 'l2', 'elasticnet', 'none']
+        , 'clf__solver' : ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']
+        }      
+#%% DecisionTreeClassifier()
+
+    gs_dt = Pipeline((
+    ('pre'  , MinMaxScaler())
+    , ('clf', DecisionTreeClassifier(**rs
+                                     , **njobs))
+    ))
+    gs_dt_params = {
+            'clf__max_depth': [ 2,  4,  6, 8, 10]
+            , 'clf__criterion':['gini','entropy']
+            , "clf__max_features":["auto", None]
+            , "clf__max_leaf_nodes":[10,20,30,40]
+            }    
+#%% KNeighborsClassifier()
+#https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html
+    gs_knn = Pipeline((
+    ('pre' , MinMaxScaler())
+    ,('clf', KNeighborsClassifier(**rs
+                                  , **njobs))
+    ))
+    
+    gs_knn_params = {
+    'clf__n_neighbors': [3, 7, 10]
+    #, 'clf__n_neighbors': range(1, 21, 2)
+    ,'clf__metric'     : ['euclidean', 'manhattan', 'minkowski']
+    , 'clf__weights'    : ['uniform', 'distance']
+    }
+#%% RandomForestClassifier()
+
+    gs_rf = Pipeline((
+    ('pre' , MinMaxScaler())
+    ,('clf', RandomForestClassifier(**rs
+                                    , **njobs
+                                    , bootstrap = True
+                                    , oob_score = True))
+    ))
+    gs_rf_params = {
+        'clf__max_depth': [4, 6, 8, 10, 12, 16, 20, None]
+        , 'clf__class_weight':['balanced','balanced_subsample']
+        , 'clf__n_estimators': [10, 100, 1000]
+        , 'clf__criterion': ['gini', 'entropy']
+        , 'clf__max_features': ['auto', 'sqrt']
+        , 'clf__min_samples_leaf': [2, 4, 8, 50]
+        , 'clf__min_samples_split': [10, 20]
+        }
+#%% XGBClassifier()
+# https://stackoverflow.com/questions/34674797/xgboost-xgbclassifier-defaults-in-python
+# https://stackoverflow.com/questions/34674797/xgboost-xgbclassifier-defaults-in-python
+    gs_xgb = Pipeline((
+    ('pre' , MinMaxScaler())
+    ,('clf', XGBClassifier(**rs
+                           , **njobs))
+    ))
+    
+    gs_xgb_params = {
+        'clf__learning_rate': [0.01, 0.05, 0.1, 0.2]
+        , 'clf__max_depth': [4, 6, 8, 10, 12, 16, 20]
+        , 'clf__min_samples_leaf': [4, 8, 12, 16, 20]
+        , 'clf__max_features': ['auto', 'sqrt']
+        }  
+#%% MLPClassifier()
+# https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html
+    gs_mlp = Pipeline((
+    ('pre' , MinMaxScaler())
+    ,('clf', MLPClassifier(**rs
+                           , **njobs
+                           , max_iter = 500))
+      ))
+
+    gs_mlp_params = {
+        'clf__hidden_layer_sizes': [(1), (2), (3)]
+        , 'clf__max_features': ['auto', 'sqrt']
+        , 'clf__min_samples_leaf': [2, 4, 8]
+        , 'clf__min_samples_split': [10, 20]
+        }
+#%% RidgeClassifier()
+# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.RidgeClassifier.html
+    gs_rc = Pipeline((
+    ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
+    ,('clf', RidgeClassifier(**rs
+                           , **njobs))
+      ))
+
+    gs_rc_params = {
+        'clf__alpha': [0.1, 0.2, 0.5, 0.8, 1.0]
+        }
+
+#%% SVC()
+# https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html
+    gs_svc = Pipeline((
+    ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
+    ,('clf', SVC(**rs
+                 , **njobs))
+      ))
+
+    gs_svc_params = {
+        'clf__kernel': ['linear', 'poly', 'rbf', 'sigmoid', 'precomputed'} 
+       , 'clf__C'    : [50, 10, 1.0, 0.1, 0.01]
+       , 'clf__gamma': ['scale', 'auto'] }
+
+#%% BaggingClassifier()
+#https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.BaggingClassifier.html
+    gs_bdt = Pipeline((
+    ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
+    ,('clf', BaggingClassifier(**rs
+                               , **njobs
+                               , bootstrap = True
+                               , oob_score = True))
+      ))
+
+    gs_bdt_params = {
+        'clf__n_estimators'    : [10, 100, 1000]
+       # If None, then the base estimator is a DecisionTreeClassifier.
+       , 'clf__base_estimator' : ['None', 'SVC()', 'KNeighborsClassifier()']# if none, DT is used
+       , 'clf__gamma': ['scale', 'auto'] }
+#%% GradientBoostingClassifier()
+# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html
+    gs_gb = Pipeline((
+    ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
+    ,('clf', GradientBoostingClassifier(**rs))
+      ))
+
+    gs_bdt_params = {
+        'clf__n_estimators'   : [10, 100, 1000]
+        , 'clf__n_estimators' : [10, 100, 1000]
+        , 'clf__learning_rate': [0.001, 0.01, 0.1]
+        , 'clf__subsample'    : [0.5, 0.7, 1.0]
+        , 'clf__max_depth'     : [3, 7, 9]
+        }
+#%% AdaBoostClassifier()
+#https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html#sklearn.ensemble.AdaBoostClassifier
+    gs_gb = Pipeline((
+    ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
+    ,('clf', AdaBoostClassifier(**rs))
+      ))
+
+    gs_bdt_params = {
+        'clf__n_estimators': [none, 1, 2]
+       , 'clf__base_estiamtor'  : ['None', 1*SVC(), 1*KNeighborsClassifier()]
+       #, 'clf___splitter'  :   ["best", "random"]
+        }
+#%% GaussianProcessClassifier()
+# https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.GaussianProcessClassifier.html
+    #GaussianProcessClassifier(1.0 * RBF(1.0)),
+    gs_gpc = Pipeline((
+    ('pre' , MinMaxScaler()) # CHECK if it wants -1 to 1
+    ,('clf', GaussianProcessClassifier(**rs))
+      ))
+
+    gs_gpc_params = {
+        'clf__kernel': [1*RBF(), 1*DotProduct(), 1*Matern(), 1*RationalQuadratic(), 1*WhiteKernel()]
+        }
+
+#%% GaussianNB()
+# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html
+    gs_gnb = Pipeline((
+    ('pre' , MinMaxScaler())
+    , ('pca', PCA() )# CHECK if it wants -1 to 1
+    ,('clf', GaussianNB(**rs))
+      ))
+
+    gs_gnb_params = {
+        'clf__priors': [None]
+        , 'clf__var_smoothing': np.logspace(0,-9, num=100)
+        }
+
+#%% QuadraticDiscriminantAnalysis()
+#https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html
+    
+    gs_qda = Pipeline((
+    ('pre' , MinMaxScaler())
+    #, ('pca', PCA() )# CHECK if it wants -1 to 1
+    ,('clf', QuadraticDiscriminantAnalysis())
+      ))
+#%% BernoulliNB()
+# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.BernoulliNB.html
+    gs_gnb = Pipeline((
+    ('pre' , MinMaxScaler())
+    ,('clf', BernoulliNB())
+      ))
+BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True)
+    gs_gnb_params = {
+        'clf__alpha': [0, 1]
+        , 'clf__binarize':['None', 0]
+        , 'clf__fit_prior': [True]
+        , 'clf__class_prior': ['None']
+        }
\ No newline at end of file
diff --git a/hyperparams.py b/hyperparams.py
index e51324d..546ab6f 100644
--- a/hyperparams.py
+++ b/hyperparams.py
@@ -16,10 +16,9 @@ scoring_refit = {'scoring': mcc_score_fn
                  ,'refit': 'mcc'}
 #n_jobs = 10 # my desktop has 12 cores
 njobs = {'n_jobs': 10}
-skf_cv = StratifiedKFold(n_splits=10,shuffle  = True)
+skf_cv = StratifiedKFold(n_splits = 10, shuffle  = True)
 #cv = {'cv': 10}
 
-
 gs_dt = GridSearchCV(estimator=DecisionTreeClassifier(**rs
                                                       #,class_weight = {1:10, 0:1}
                                                       ),
@@ -43,8 +42,8 @@ gs_dt_fit = gs_dt.fit(num_df_wtgt[numerical_FN]
 
 gs_dt_fit_res  = gs_dt_fit.cv_results_
 
-print('Best model:\n', gs_dt.best_params_)
-print('Best models score:\n', gs_dt.best_score_)
+print('Best model:\n'             , gs_dt.best_params_)
+print('Best models score:\n'      , gs_dt.best_score_)
 print('Check best models score:\n', mean(gs_dt_fit_res['mean_test_mcc']))
 
 #%% Check the scores:
@@ -106,3 +105,104 @@ means = clf.cv_results_['mean_test_score']
 stds = clf.cv_results_['std_test_score']
 for mean, std, params in zip(means, stds, clf.cv_results_['params']):
     print("%0.3f (+/-%0.03f) for %r" % (mean, std * 2, params))
+    
+########################################################################    
+#%%: Hyperparams with SKF and trying different scoring functions
+ # https://stackoverflow.com/questions/57248072/gridsearchcv-gives-different-result
+#  https://stackoverflow.com/questions/44947574/what-is-the-meaning-of-mean-test-score-in-cv-result
+#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn
+
+
+#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn
+# If you only want accuracy, then you can simply use cross_val_score()
+# kf = KFold(n_splits=10)
+# clf_tree=DecisionTreeClassifier()
+# scores = cross_val_score(clf_tree, X, y, cv=kf)
+# avg_score = np.mean(score_array)
+# print(avg_score)
+# Here cross_val_score will take as input your original X and y (without splitting into train and test). cross_val_score will automatically split them into train and test, fit the model on train data and score on test data. And those scores will be returned in the scores variable.
+# So when you have 10 folds, 10 scores will be returned in scores variable. You can then just take an average of that.
+
+mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
+
+scoring_refit_recall = {'scoring': 'recall'
+                 ,'refit': 'recall'}
+
+scoring_refit_recall = {'scoring': 'precision'
+                 ,'refit': 'precision'}
+
+scoring_refit_mcc = {'scoring': mcc_score_fn
+                 ,'refit': 'mcc'}
+#n_jobs = 10 # my desktop has 12 cores
+#cv = {'cv': 10}#%%
+
+njobs = {'n_jobs': 10}
+skf_cv = StratifiedKFold(n_splits = 10, shuffle  = True)
+
+#%% GSCV: RandomForest
+gs_rf = GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, oob_score = True
+                                                      #,class_weight = {1: 10/11, 0: 1/11}
+                                                      )
+                     , param_grid=[{'max_depth': [4, 6, 8, 10, None]
+                                 , 'max_features': ['auto', 'sqrt']
+                                 , 'min_samples_leaf': [2, 4, 8]
+                                 , 'min_samples_split': [10, 20]}]
+                    , cv = skf_cv
+                    , **njobs
+                    , **scoring_refit_recall
+                    #, **scoring_refit_mcc
+                    #, scoring = scoring_fn, refit  = False
+                    )
+#gs_rf.fit(X_train, y_train)
+#gs_rf_fit = gs_rf.fit(X_train y_train)
+
+gs_rf.fit(X, y)
+gs_rf_fit = gs_rf.fit(X, y)
+gs_rf_res = gs_rf_fit.cv_results_
+print('Best model:\n', gs_rf.best_params_)
+print('Best models score:\n', gs_rf.best_score_)
+print('Check mean models score:\n', mean(gs_rf_res['mean_test_score']))
+
+#%% Proof of concept: manual inspection to see how best score is calcualted!
+# SATISFIED!
+# Best_model example: recall, Best model's score:  0.8059288537549408
+# {'max_depth': 4, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 10}
+
+# Best model example: mcc, Best models score: 0.42504894661702863
+# {'max_depth': 4, 'max_features': 'auto', 'min_samples_leaf': 4, 'min_samples_split': 20}
+ 
+# Best model example: precision, Best models score: 0.7144745254745255
+# {'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10}
+
+best_model = [{'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10 }]
+
+gs_results_df = pd.DataFrame(gs_rf_res)
+gs_results_df.shape
+
+gs_best_df = gs_results_df.loc[gs_results_df['params'].isin(best_model)]
+gs_best_df.shape
+
+gs_best_df_test = gs_best_df.filter(like = 'test_', axis = 1)
+gs_best_df_test.shape
+
+gs_best_df_test_recall = gs_best_df_test.filter(like = '_score', axis = 1)
+gs_best_df_test_recall.shape
+
+f = gs_best_df_test_recall.filter(like='split', axis = 1)
+f.shape
+#gs_best_df_test_mcc = gs_best_df_test.filter(like = '_mcc', axis = 1)
+#f = gs_best_df_test_mcc.filter(like='split', axis = 1)
+
+f.iloc[:,[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]].mean(axis = 1)
+# recall: 0.801186 vs  0.8059288537549408
+# mcc: 0.425049 vs 0.42504894661702863
+# precision: 0.714475 vs 0.7144745254745255
+
+#%%
+
+#%% Check the scores:
+print([(len(train), len(test)) for train, test in skf_cv.split(X, y)])
+gs_rf_fit.cv_results_
+#its the weighted average!?
+#%%
+
diff --git a/hyperparams_p1.py b/hyperparams_p1.py
index 4b29506..b4dfdb7 100644
--- a/hyperparams_p1.py
+++ b/hyperparams_p1.py
@@ -1,106 +1,158 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-Created on Wed Mar 16 16:55:06 2022
+Created on Sun Mar 20 13:02:54 2022
 
 @author: tanu
 """
-# https://stackoverflow.com/questions/57248072/gridsearchcv-gives-different-result
-#  https://stackoverflow.com/questions/44947574/what-is-the-meaning-of-mean-test-score-in-cv-result
-#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn
+# https://machinelearningmastery.com/hyperparameters-for-classification-machine-learning-algorithms/
 
+#%% LogisticRegression
+# example of grid searching key hyperparametres for logistic regression
+from sklearn.datasets import make_blobs
+from sklearn.model_selection import RepeatedStratifiedKFold
+from sklearn.model_selection import GridSearchCV
+from sklearn.linear_model import LogisticRegression
+# define dataset
+X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)
+# define models and parameters
+model = LogisticRegression()
+solvers = ['newton-cg', 'lbfgs', 'liblinear']
+penalty = ['l2']
+c_values = [100, 10, 1.0, 0.1, 0.01]
+# define grid search
+grid = dict(solver=solvers,penalty=penalty,C=c_values)
+cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
+grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0)
+grid_result = grid_search.fit(X, y)
+# summarize results
+print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
+means = grid_result.cv_results_['mean_test_score']
+stds = grid_result.cv_results_['std_test_score']
+params = grid_result.cv_results_['params']
+for mean, stdev, param in zip(means, stds, params):
+    print("%f (%f) with: %r" % (mean, stdev, param))
+#%% RidgeClassifier
+from sklearn.datasets import make_blobs
+from sklearn.model_selection import RepeatedStratifiedKFold
+from sklearn.model_selection import GridSearchCV
+from sklearn.linear_model import RidgeClassifier
+# define dataset
+X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)
+# define models and parameters
+model = RidgeClassifier()
+alpha = [0.9, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.1, 1.0]
+# define grid search
+grid = dict(alpha=alpha)
+cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
+grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0)
+grid_result = grid_search.fit(X, y)
+# summarize results
+print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
+means = grid_result.cv_results_['mean_test_score']
+stds = grid_result.cv_results_['std_test_score']
+params = grid_result.cv_results_['params']
+for mean, stdev, param in zip(means, stds, params):
+    print("%f (%f) with: %r" % (mean, stdev, param))
+    
+# NOTES:
+# alpha: If all alphas return the same mean, which do you chose?
+# Python seems to chose the first one?  
+#  https://stats.stackexchange.com/questions/166950/alpha-parameter-in-ridge-regression-is-high
+# The L2 norm term in ridge regression is weighted by the regularization parameter
+# alpha. So, the alpha parameter need not be small. But, for a larger alpha, the 
+# flexibility of the fit would be very strict.
 
-#https://stackoverflow.com/questions/47257952/how-to-get-average-score-of-k-fold-cross-validation-with-sklearn
-# If you only want accuracy, then you can simply use cross_val_score()
-# kf = KFold(n_splits=10)
-# clf_tree=DecisionTreeClassifier()
-# scores = cross_val_score(clf_tree, X, y, cv=kf)
-# avg_score = np.mean(score_array)
-# print(avg_score)
-# Here cross_val_score will take as input your original X and y (without splitting into train and test). cross_val_score will automatically split them into train and test, fit the model on train data and score on test data. And those scores will be returned in the scores variable.
-# So when you have 10 folds, 10 scores will be returned in scores variable. You can then just take an average of that.
+# So, if the alpha value is 0, it means that it is just an Ordinary Least Squares
+# Regression model. So, the larger is the alpha, the higher is the smoothness constraint.
+# So, the smaller the value of alpha, the higher would be the magnitude of the coefficients.
 
-mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
+# Could be that the model does not fit very well. With a very large alpha, 
+# the algorithm more or else ignores the IV's and fits a mean. – Placidia
+# @Placidia, yes I would completely agree with your comment. I was just trying to
+# explain the significance of alpha as a parameter (as asked in the question) in
+# Ridge Regression, and how it's change would affect the fit and the coefficients.
+# Thank you for including the point in the comment.
+# ** READ: https://machinelearningcompass.com/machine_learning_models/ridge_regression/
+#%% KNeighborsClassifier
+from sklearn.datasets import make_blobs
+from sklearn.model_selection import RepeatedStratifiedKFold
+from sklearn.model_selection import GridSearchCV
+from sklearn.neighbors import KNeighborsClassifier
+# define dataset
+X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)
+# define models and parameters
+model = KNeighborsClassifier()
+n_neighbors = range(1, 21, 2)
+weights = ['uniform', 'distance']
+metric = ['euclidean', 'manhattan', 'minkowski']
+#p = [1,2]
+# define grid search
+grid = dict(n_neighbors=n_neighbors,weights=weights,metric=metric)
+cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
+grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0)
+grid_result = grid_search.fit(X, y)
+# summarize results
+print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
+means = grid_result.cv_results_['mean_test_score']
+stds = grid_result.cv_results_['std_test_score']
+params = grid_result.cv_results_['params']
+for mean, stdev, param in zip(means, stds, params):
+    print("%f (%f) with: %r" % (mean, stdev, param))
+    
+# NOTES:
+# https://vitalflux.com/k-nearest-neighbors-explained-with-python-examples/
+# https://vitalflux.com/overfitting-underfitting-concepts-interview-questions/
+# Larger value of K ==> model may underfit
+# Smaller value of K ==> the model may overfit. 
+#%%Support Vector Machine (SVM)
+# example of grid searching key hyperparametres for SVC
+from sklearn.datasets import make_blobs
+from sklearn.model_selection import RepeatedStratifiedKFold
+from sklearn.model_selection import GridSearchCV
+from sklearn.svm import SVC
+# define dataset
+X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)
+# define model and parameters
+model = SVC()
+kernel = ['poly', 'rbf', 'sigmoid']
+C = [50, 10, 1.0, 0.1, 0.01]
+gamma = ['scale']
+# define grid search
+grid = dict(kernel=kernel,C=C,gamma=gamma)
+cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
+grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy',error_score=0)
+grid_result = grid_search.fit(X, y)
+# summarize results
+print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
+means = grid_result.cv_results_['mean_test_score']
+stds = grid_result.cv_results_['std_test_score']
+params = grid_result.cv_results_['params']
+for mean, stdev, param in zip(means, stds, params):
+    print("%f (%f) with: %r" % (mean, stdev, param))
+    
+# NOTES:
+# https://stats.stackexchange.com/questions/31066/what-is-the-influence-of-c-in-svms-with-linear-kernel
+# SVM terms: hyperplane, C and soft margins
+# hyperplane that can min(max(dist)) of the suppor vectors from tne hyperplane
+# High C ==> increase overfitting
+# Low C ==> increase underfitting
 
-scoring_refit_recall = {'scoring': 'recall'
-                 ,'refit': 'recall'}
+# But if C is a regularization parameter, why does a high C increase 
+# overfitting, when generally speaking regularization is done to
+# mitigate overfitting, i.e., by creating a more general model? 
+# C is a regularisation parameter, but it is essentially attached to 
+# the data misfit term (the sum of the slack variables) rather than 
+# the regularisation term (the margin bit), so a larger value of C 
+# means less regularisation, rather than more. Alternatively you can
+# view the usual representation of the rgularisation parameter 
+# as 1/C.
 
-scoring_refit_recall = {'scoring': 'precision'
-                 ,'refit': 'precision'}
-
-scoring_refit_mcc = {'scoring': mcc_score_fn
-                 ,'refit': 'mcc'}
-#n_jobs = 10 # my desktop has 12 cores
-#cv = {'cv': 10}#%%
-
-njobs = {'n_jobs': 10}
-skf_cv = StratifiedKFold(n_splits = 10, shuffle  = True)
-
-#%% GSCV: RandomForest
-gs_rf = GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, oob_score = True
-                                                      #,class_weight = {1: 10/11, 0: 1/11}
-                                                      )
-                     , param_grid=[{'max_depth': [4, 6, 8, 10, None]
-                                 , 'max_features': ['auto', 'sqrt']
-                                 , 'min_samples_leaf': [2, 4, 8]
-                                 , 'min_samples_split': [10, 20]}]
-                    , cv = skf_cv
-                    , **njobs
-                    , **scoring_refit_recall
-                    #, **scoring_refit_mcc
-                    #, scoring = scoring_fn, refit  = False
-                    )
-#gs_rf.fit(X_train, y_train)
-#gs_rf_fit = gs_rf.fit(X_train y_train)
-
-gs_rf.fit(X, y)
-gs_rf_fit = gs_rf.fit(X, y)
-gs_rf_res = gs_rf_fit.cv_results_
-print('Best model:\n', gs_rf.best_params_)
-print('Best models score:\n', gs_rf.best_score_)
-print('Check mean models score:\n', mean(gs_rf_res['mean_test_score']))
-
-#%% Proof of concept: manual inspection to see how best score is calcualted!
-# SATISFIED!
-# Best_model example: recall, Best model's score:  0.8059288537549408
-# {'max_depth': 4, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 10}
-
-# Best model example: mcc, Best models score: 0.42504894661702863
-# {'max_depth': 4, 'max_features': 'auto', 'min_samples_leaf': 4, 'min_samples_split': 20}
- 
-# Best model example: precision, Best models score: 0.7144745254745255
-# {'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10}
-
-best_model = [{'max_depth': 6, 'max_features': 'sqrt', 'min_samples_leaf': 8, 'min_samples_split': 10 }]
-
-gs_results_df = pd.DataFrame(gs_rf_res)
-gs_results_df.shape
-
-gs_best_df = gs_results_df.loc[gs_results_df['params'].isin(best_model)]
-gs_best_df.shape
-
-gs_best_df_test = gs_best_df.filter(like = 'test_', axis = 1)
-gs_best_df_test.shape
-
-gs_best_df_test_recall = gs_best_df_test.filter(like = '_score', axis = 1)
-gs_best_df_test_recall.shape
-
-f = gs_best_df_test_recall.filter(like='split', axis = 1)
-f.shape
-#gs_best_df_test_mcc = gs_best_df_test.filter(like = '_mcc', axis = 1)
-#f = gs_best_df_test_mcc.filter(like='split', axis = 1)
-
-f.iloc[:,[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]].mean(axis = 1)
-# recall: 0.801186 vs  0.8059288537549408
-# mcc: 0.425049 vs 0.42504894661702863
-# precision: 0.714475 vs 0.7144745254745255
-
-#%%
-
-#%% Check the scores:
-print([(len(train), len(test)) for train, test in skf_cv.split(X, y)])
-gs_rf_fit.cv_results_
-#its the weighted average!?
-#%%
+#C is a regularization parameter that controls the trade off 
+#between the achieving a low training error and a low testing
+# error that is the ability to generalize your classifier to unseen data.
 
+# C Parameter is used for controlling the outliers:
+# low C implies ==> we are allowing more outliers
+# high C implies we are allowing fewer outliers.
 
diff --git a/imports.py b/imports.py
index b5938a3..c6a2531 100644
--- a/imports.py
+++ b/imports.py
@@ -81,6 +81,11 @@ njobs = {'n_jobs': 10}
 skf_cv = StratifiedKFold(n_splits = 10
                           #, shuffle = False, random_state= None)
                            , shuffle = True,**rs)
+rskf_cv = RepeatedStratifiedKFold(n_splits = 10
+                                  , n_repeats=3
+                                 #, shuffle = False, random_state= None)
+                                 #, shuffle = True
+                                 ,**rs)
 #my_mcc = make_scorer({'mcc':make_scorer(matthews_corrcoef})
 mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
 
diff --git a/intra_model_gscv.py b/intra_model_gscv.py
index 40ef39e..a4f905c 100644
--- a/intra_model_gscv.py
+++ b/intra_model_gscv.py
@@ -37,7 +37,7 @@ class ClfSwitcher(BaseEstimator):
     #def recall_score(self, X, y):
     #    return self.estimator.recall_score(X, y)
 #%% Custom GridSearch: IntraModel[orig]
-def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical','mixed']) :
+def grid_search(input_df, target, sel_cv, var_type = ['numerical', 'categorical','mixed']) :
     
     pipeline1 = Pipeline((
     ('pre', MinMaxScaler())
@@ -73,7 +73,7 @@ def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical
     for i in range(len(pars)):
         print('IIIII===>', i)
         gs = GridSearchCV(pips[i], pars[i]
-                          , cv = skf_cv
+                          , cv = sel_cv
                           , **scoring_refit
                           #, refit=False
                           , **njobs
@@ -82,9 +82,21 @@ def grid_search2(input_df, target, skf_cv, var_type = ['numerical', 'categorical
         print ("finished Gridsearch")
         print ('\nBest model:', gs.best_params_)
         print ('\nBest score:', gs.best_score_)
-#%% Custom grid_search: Intra-Model [with return]
+# TODO: add
+#         # summarize results
+#         print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
+#         means = grid_result.cv_results_['mean_test_score']
+#         stds = grid_result.cv_results_['std_test_score']
+#         params = grid_result.cv_results_['params']
+#         for mean, stdev, param in zip(means, stds, params):
+#             print("%f (%f) with: %r" % (mean, stdev, param))
+
+# CALL: grid_search [orig]
+grid_search()
+
+# #%% Custom grid_search: Intra-Model [with return]
 def grid_search(input_df, target
-                , skf_cv
+                , sel_cv
                 , chosen_scoreD #scoring_refit 
                 #, var_type = ['numerical', 'categorical','mixed']
                 ):
@@ -128,7 +140,7 @@ def grid_search(input_df, target
     
         print("\nStarting Gridsearch for model:", model_name, i)
         gs = GridSearchCV(all_pipelines[i], all_parameters[i]
-                          , cv = skf_cv
+                          , cv = sel_cv
                           #, **scoring_refit
                           #, refit=False
                           , **chosen_scoreD
@@ -150,6 +162,9 @@ def grid_search(input_df, target
         out[model_name].update(chosen_scoreD.copy())
         out[model_name].update({'best_score': gs.best_score_}.copy())
     return(out)
+
+# TODO: 
+# print, or see for each model mean test score and sd, sometimes they can be identical and your best model just picks one!
 #%% call CUSTOM grid_search: INTRA model [with return]
 # call
 chosen_score =  {'scoring': 'recall'
@@ -158,7 +173,6 @@ mcc_score_fn = {'chosen_scoreD': {'scoring': {'mcc': make_scorer(matthews_corrco
                                             ,'refit': 'mcc'}
                                   }
                 }
-
 intra_models = grid_search(X, y
             , skf_cv = skf_cv
             , chosen_scoreD= chosen_score
diff --git a/loopity_loop.py b/loopity_loop.py
index a0afc35..077ca66 100644
--- a/loopity_loop.py
+++ b/loopity_loop.py
@@ -40,7 +40,7 @@ njobs = {'n_jobs': 10}
 # TODO: get accuracy and other scores through K-fold cv
 
 # Multiple Classification - Model Pipeline
-def MultClassPipeSKFLoop(input_df, target, skf_cv, var_type = ['numerical','categorical','mixed']):
+def MultClassPipeSKFLoop(input_df, target, sel_cv, var_type = ['numerical','categorical','mixed']):
 
     '''
     @ param input_df: input features 
@@ -131,7 +131,7 @@ def MultClassPipeSKFLoop(input_df, target, skf_cv, var_type = ['numerical','cate
         fold_dict.update({ model_name: {}})
 
     #scores_df = pd.DataFrame()
-    for train_index, test_index in skf_cv.split(input_df, target):
+    for train_index, test_index in sel_cv.split(input_df, target):
         x_train_fold, x_test_fold = input_df.iloc[train_index], input_df.iloc[test_index]
         y_train_fold, y_test_fold = target.iloc[train_index], target.iloc[test_index]
         #print("Fold: ", fold_no, len(train_index), len(test_index))
diff --git a/loopity_loop_CALL.py b/loopity_loop_CALL.py
index e70763e..e510a42 100644
--- a/loopity_loop_CALL.py
+++ b/loopity_loop_CALL.py
@@ -6,16 +6,17 @@ Created on Fri Mar 11 11:15:50 2022
 @author: tanu
 """
 #%% variables
-rs = {'random_state': 42}
+# rs = {'random_state': 42}
 
-skf_cv = StratifiedKFold(n_splits = 10
-                          #, shuffle = False, random_state= None)
-                          , shuffle = True,**rs)
+# skf_cv = StratifiedKFold(n_splits = 10
+#                           #, shuffle = False, random_state= None)
+#                           , shuffle = True,**rs)
 #%% MultClassPipeSKFLoop: function call()
 t3_res = MultClassPipeSKFLoop(input_df = num_df_wtgt[numerical_FN]
                           , target = num_df_wtgt['mutation_class']
                           , var_type = 'numerical'
-                          , skf_cv = skf_cv)
+                          , sel_cv = skf_cv)
+                          #, sel_cv = rskf_cv)
 pp.pprint(t3_res)
 #print(t3_res)
 ################################################################