Here is a comparison between with and without iid for a cross-validation classification on iris:

from sklearn.utils import shuffle
from sklearn.metrics import SCORERS
from sklearn.model_selection import GridSearchCV, KFold
from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_iris
import numpy as np


X, y = load_iris(return_X_y=True)
X, y = shuffle(X, y)
X = X[:145]
y = y[:145]

clf = KNeighborsClassifier()
same_score_list = []
diff_score_list = []
for name, scorer in SCORERS.items():
    try:
        kfold = KFold(random_state=0)

        meta_iid = GridSearchCV(clf, {'n_neighbors': [2, 3, 4]}, iid=True,
                                scoring=scorer, cv=kfold)
        meta_iid.fit(X, y)
        meta_non_iid = GridSearchCV(clf, {'n_neighbors': [2, 3, 4]},
                                    iid=False, scoring=scorer, cv=kfold)
        meta_non_iid.fit(X, y)
        same_score = np.allclose(meta_iid.cv_results_['mean_test_score'],
                                      meta_non_iid.cv_results_['mean_test_score'])
        if same_score :
            same_score_list.append(name)
        else:
            diff_score_list.append(name)
    except:
        pass
print('same score: %s' % str(same_score_list))
print('different score: %s' % str(diff_score_list))

Returns:

same score: ['neg_median_absolute_error']
different score: ['explained_variance', 'r2', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'accuracy', 'neg_log_loss', 'adjusted_rand_score', 'homogeneity_score', 'completeness_score', 'v_measure_score', 'mutual_info_score', 'adjusted_mutual_info_score', 'normalized_mutual_info_score', 'fowlkes_mallows_score', 'precision_macro', 'precision_micro', 'precision_weighted', 'recall_macro', 'recall_micro', 'recall_weighted', 'f1_macro', 'f1_micro', 'f1_weighted']

Uh oh!

deprecation warnings in examples for iid and train_size #11559

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions