diff --git a/Makefile b/Makefile index 20bc4f582e124..8ee4aa14f2b0b 100644 --- a/Makefile +++ b/Makefile @@ -26,9 +26,6 @@ clean: clean-ctags in: inplace # just a shortcut inplace: - # to avoid errors in 0.15 upgrade - rm -f sklearn/utils/sparsefuncs*.so - rm -f sklearn/utils/random*.so $(PYTHON) setup.py build_ext -i test-code: in diff --git a/README.rst b/README.rst index 8d67f2017c1a5..ff9ec69e9baf6 100644 --- a/README.rst +++ b/README.rst @@ -78,7 +78,7 @@ Development We welcome new contributors of all experience levels. The scikit-learn community goals are to be helpful, welcoming, and effective. The -`Contributor's Guide `_ +`Development Guide `_ has detailed information about contributing code, documentation, tests, and more. We've included some basic information in this README. @@ -86,7 +86,7 @@ Important links ~~~~~~~~~~~~~~~ - Official source code repo: https://github.com/scikit-learn/scikit-learn -- Download releases: http://sourceforge.net/projects/scikit-learn/files/ +- Download releases: https://pypi.python.org/pypi/scikit-learn - Issue tracker: https://github.com/scikit-learn/scikit-learn/issues Source code @@ -158,4 +158,4 @@ Communication - Mailing list: https://mail.python.org/mailman/listinfo/scikit-learn - IRC channel: ``#scikit-learn`` at ``irc.freenode.net`` - Stack Overflow: http://stackoverflow.com/questions/tagged/scikit-learn -- Website: http://scikit-learn.org \ No newline at end of file +- Website: http://scikit-learn.org diff --git a/appveyor.yml b/appveyor.yml index 205018f166bf6..8d3b3e7d05b19 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -36,6 +36,16 @@ environment: install: + # If there is a newer build queued for the same PR, cancel this one. + # The AppVeyor 'rollout builds' option is supposed to serve the same + # purpose but is problematic because it tends to cancel builds pushed + # directly to master instead of just PR builds. + # credits: JuliaLang developers. + - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` + https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` + Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` + throw "There are newer queued builds for this pull request, failing early." } + # Install Python (from the official .msi of http://python.org) and pip when # not already installed. - "powershell ./build_tools/appveyor/install.ps1" diff --git a/benchmarks/bench_lof.py b/benchmarks/bench_lof.py new file mode 100644 index 0000000000000..620adc3d43b0c --- /dev/null +++ b/benchmarks/bench_lof.py @@ -0,0 +1,119 @@ +""" +============================ +LocalOutlierFactor benchmark +============================ + +A test of LocalOutlierFactor on classical anomaly detection datasets. + +""" + +from time import time +import numpy as np +import matplotlib.pyplot as plt +from sklearn.neighbors import LocalOutlierFactor +from sklearn.metrics import roc_curve, auc +from sklearn.datasets import fetch_kddcup99, fetch_covtype, fetch_mldata +from sklearn.preprocessing import LabelBinarizer +from sklearn.utils import shuffle as sh + +print(__doc__) + +np.random.seed(2) + +# datasets available: ['http', 'smtp', 'SA', 'SF', 'shuttle', 'forestcover'] +datasets = ['shuttle'] + +novelty_detection = True # if False, training set polluted by outliers + +for dataset_name in datasets: + # loading and vectorization + print('loading data') + if dataset_name in ['http', 'smtp', 'SA', 'SF']: + dataset = fetch_kddcup99(subset=dataset_name, shuffle=True, + percent10=False) + X = dataset.data + y = dataset.target + + if dataset_name == 'shuttle': + dataset = fetch_mldata('shuttle') + X = dataset.data + y = dataset.target + X, y = sh(X, y) + # we remove data with label 4 + # normal data are then those of class 1 + s = (y != 4) + X = X[s, :] + y = y[s] + y = (y != 1).astype(int) + + if dataset_name == 'forestcover': + dataset = fetch_covtype(shuffle=True) + X = dataset.data + y = dataset.target + # normal data are those with attribute 2 + # abnormal those with attribute 4 + s = (y == 2) + (y == 4) + X = X[s, :] + y = y[s] + y = (y != 2).astype(int) + + print('vectorizing data') + + if dataset_name == 'SF': + lb = LabelBinarizer() + lb.fit(X[:, 1]) + x1 = lb.transform(X[:, 1]) + X = np.c_[X[:, :1], x1, X[:, 2:]] + y = (y != 'normal.').astype(int) + + if dataset_name == 'SA': + lb = LabelBinarizer() + lb.fit(X[:, 1]) + x1 = lb.transform(X[:, 1]) + lb.fit(X[:, 2]) + x2 = lb.transform(X[:, 2]) + lb.fit(X[:, 3]) + x3 = lb.transform(X[:, 3]) + X = np.c_[X[:, :1], x1, x2, x3, X[:, 4:]] + y = (y != 'normal.').astype(int) + + if dataset_name == 'http' or dataset_name == 'smtp': + y = (y != 'normal.').astype(int) + + n_samples, n_features = np.shape(X) + n_samples_train = n_samples // 2 + n_samples_test = n_samples - n_samples_train + + X = X.astype(float) + X_train = X[:n_samples_train, :] + X_test = X[n_samples_train:, :] + y_train = y[:n_samples_train] + y_test = y[n_samples_train:] + + if novelty_detection: + X_train = X_train[y_train == 0] + y_train = y_train[y_train == 0] + + print('LocalOutlierFactor processing...') + model = LocalOutlierFactor(n_neighbors=20) + tstart = time() + model.fit(X_train) + fit_time = time() - tstart + tstart = time() + + scoring = -model.decision_function(X_test) # the lower, the more normal + predict_time = time() - tstart + fpr, tpr, thresholds = roc_curve(y_test, scoring) + AUC = auc(fpr, tpr) + plt.plot(fpr, tpr, lw=1, + label=('ROC for %s (area = %0.3f, train-time: %0.2fs,' + 'test-time: %0.2fs)' % (dataset_name, AUC, fit_time, + predict_time))) + +plt.xlim([-0.05, 1.05]) +plt.ylim([-0.05, 1.05]) +plt.xlabel('False Positive Rate') +plt.ylabel('True Positive Rate') +plt.title('Receiver operating characteristic') +plt.legend(loc="lower right") +plt.show() diff --git a/circle.yml b/circle.yml index 2e4fa5c3a9370..74fd2fb53c90b 100644 --- a/circle.yml +++ b/circle.yml @@ -6,7 +6,8 @@ dependencies: # Check whether the doc build is required, install build dependencies and # run sphinx to build the doc. override: - - ./build_tools/circle/build_doc.sh + - ./build_tools/circle/build_doc.sh: + timeout: 3600 # seconds test: # Grep error on the documentation override: diff --git a/doc/conf.py b/doc/conf.py index 9c75922b2a50a..e60fc167f9d49 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -35,6 +35,7 @@ 'numpy_ext.numpydoc', 'sphinx.ext.linkcode', 'sphinx.ext.doctest', 'sphinx_gallery.gen_gallery', + 'sphinx_issues', ] # pngmath / imgmath compatibility layer for different sphinx versions @@ -269,6 +270,13 @@ def make_carousel_thumbs(app, exception): sphinx_gallery.gen_rst.scale_image(image, c_thumb, max_width, 190) +# Config for sphinx_issues + +issues_uri = 'https://github.com/scikit-learn/scikit-learn/issues/{issue}' +issues_github_path = 'scikit-learn/scikit-learn' +issues_user_uri = 'https://github.com/{user}' + + def setup(app): # to hide/show the prompt in code examples: app.add_javascript('js/copybutton.js') diff --git a/doc/documentation.rst b/doc/documentation.rst index 3b5bcd889ccc0..f9e2492fbcdb0 100644 --- a/doc/documentation.rst +++ b/doc/documentation.rst @@ -64,7 +64,7 @@ Documentation of scikit-learn 0.19.dev0
-

Contributing

+

Development

Information on how to contribute. This also contains useful information for advanced users, for example how to build their own estimators. diff --git a/doc/faq.rst b/doc/faq.rst index 7a4a2f2a8fd4a..3937b6a223977 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -248,10 +248,13 @@ Python processes for parallel computing. Unfortunately this is a violation of the POSIX standard and therefore some software editors like Apple refuse to consider the lack of fork-safety in Accelerate / vecLib as a bug. -In Python 3.4+ it is now possible to configure ``multiprocessing`` to use the -'forkserver' or 'spawn' start methods (instead of the default 'fork') to manage -the process pools. This makes it possible to not be subject to this issue -anymore. +In Python 3.4+ it is now possible to configure ``multiprocessing`` to +use the 'forkserver' or 'spawn' start methods (instead of the default +'fork') to manage the process pools. To work around this issue when +using scikit-learn, you can set the JOBLIB_START_METHOD environment +variable to 'forkserver'. However the user should be aware that using +the 'forkserver' method prevents joblib.Parallel to call function +interactively defined in a shell session. If you have custom code that uses ``multiprocessing`` directly instead of using it via joblib you can enable the 'forkserver' mode globally for your diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index 8a077daf018df..a4d893bafb340 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -186,6 +186,7 @@ Splitter Functions :template: function.rst model_selection.train_test_split + model_selection.check_cv Hyper-parameter optimizers -------------------------- @@ -201,6 +202,13 @@ Hyper-parameter optimizers model_selection.ParameterGrid model_selection.ParameterSampler + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + model_selection.fit_grid_point + Model validation ---------------- @@ -315,7 +323,6 @@ Samples generator decomposition.PCA decomposition.IncrementalPCA decomposition.ProjectedGradientNMF - decomposition.RandomizedPCA decomposition.KernelPCA decomposition.FactorAnalysis decomposition.FastICA @@ -560,7 +567,6 @@ From text gaussian_process.GaussianProcessRegressor gaussian_process.GaussianProcessClassifier - gaussian_process.GaussianProcess Kernels: @@ -957,7 +963,6 @@ See the :ref:`metrics` section of the user guide for further details. mixture.GaussianMixture mixture.BayesianGaussianMixture - mixture.DPGMM .. _multiclass_ref: @@ -1051,7 +1056,8 @@ See the :ref:`metrics` section of the user guide for further details. neighbors.LSHForest neighbors.DistanceMetric neighbors.KernelDensity - + neighbors.LocalOutlierFactor + .. autosummary:: :toctree: generated/ :template: function.rst @@ -1349,3 +1355,67 @@ Low-level methods utils.estimator_checks.check_estimator utils.resample utils.shuffle + + +Recently deprecated +=================== + +To be removed in 0.19 +--------------------- + +.. autosummary:: + :toctree: generated/ + :template: deprecated_class.rst + + lda.LDA + qda.QDA + +.. autosummary:: + :toctree: generated/ + :template: deprecated_function.rst + + datasets.load_lfw_pairs + datasets.load_lfw_people + + +To be removed in 0.20 +--------------------- + +.. autosummary:: + :toctree: generated/ + :template: deprecated_class.rst + + grid_search.ParameterGrid + grid_search.ParameterSampler + grid_search.GridSearchCV + grid_search.RandomizedSearchCV + cross_validation.LeaveOneOut + cross_validation.LeavePOut + cross_validation.KFold + cross_validation.LabelKFold + cross_validation.LeaveOneLabelOut + cross_validation.LeavePLabelOut + cross_validation.LabelShuffleSplit + cross_validation.StratifiedKFold + cross_validation.ShuffleSplit + cross_validation.StratifiedShuffleSplit + cross_validation.PredefinedSplit + decomposition.RandomizedPCA + gaussian_process.GaussianProcess + mixture.GMM + mixture.DPGMM + mixture.VBGMM + + +.. autosummary:: + :toctree: generated/ + :template: deprecated_function.rst + + grid_search.fit_grid_point + learning_curve.learning_curve + learning_curve.validation_curve + cross_validation.cross_val_predict + cross_validation.cross_val_score + cross_validation.check_cv + cross_validation.permutation_test_score + cross_validation.train_test_split \ No newline at end of file diff --git a/doc/modules/outlier_detection.rst b/doc/modules/outlier_detection.rst index 6682c97bced55..ee54bef985e71 100644 --- a/doc/modules/outlier_detection.rst +++ b/doc/modules/outlier_detection.rst @@ -165,9 +165,10 @@ This strategy is illustrated below. * See :ref:`sphx_glr_auto_examples_covariance_plot_outlier_detection.py` for a comparison of :class:`ensemble.IsolationForest` with + :class:`neighbors.LocalOutlierFactor`, :class:`svm.OneClassSVM` (tuned to perform like an outlier detection method) and a covariance-based outlier detection with - :class:`covariance.MinCovDet`. + :class:`covariance.EllipticEnvelope`. .. topic:: References: @@ -175,8 +176,65 @@ This strategy is illustrated below. Data Mining, 2008. ICDM'08. Eighth IEEE International Conference on. -One-class SVM versus Elliptic Envelope versus Isolation Forest --------------------------------------------------------------- +Local Outlier Factor +-------------------- +Another efficient way to perform outlier detection on moderately high dimensional +datasets is to use the Local Outlier Factor (LOF) algorithm. + +The :class:`neighbors.LocalOutlierFactor` (LOF) algorithm computes a score +(called local outlier factor) reflecting the degree of abnormality of the +observations. +It measures the local density deviation of a given data point with respect to +its neighbors. The idea is to detect the samples that have a substantially +lower density than their neighbors. + +In practice the local density is obtained from the k-nearest neighbors. +The LOF score of an observation is equal to the ratio of the +average local density of his k-nearest neighbors, and its own local density: +a normal instance is expected to have a local density similar to that of its +neighbors, while abnormal data are expected to have much smaller local density. + +The number k of neighbors considered, (alias parameter n_neighbors) is typically +chosen 1) greater than the minimum number of objects a cluster has to contain, +so that other objects can be local outliers relative to this cluster, and 2) +smaller than the maximum number of close by objects that can potentially be +local outliers. +In practice, such informations are generally not available, and taking +n_neighbors=20 appears to work well in general. +When the proportion of outliers is high (i.e. greater than 10 \%, as in the +example below), n_neighbors should be greater (n_neighbors=35 in the example +below). + +The strength of the LOF algorithm is that it takes both local and global +properties of datasets into consideration: it can perform well even in datasets +where abnormal samples have different underlying densities. +The question is not, how isolated the sample is, but how isolated it is +with respect to the surrounding neighborhood. + +This strategy is illustrated below. + +.. figure:: ../auto_examples/neighbors/images/sphx_glr_plot_lof_001.png + :target: ../auto_examples/neighbors/plot_lof.html + :align: center + :scale: 75% + +.. topic:: Examples: + + * See :ref:`sphx_glr_auto_example_neighbors_plot_lof.py` for + an illustration of the use of :class:`neighbors.LocalOutlierFactor`. + + * See :ref:`sphx_glr_auto_example_covariance_plot_outlier_detection.py` for a + comparison with other anomaly detection methods. + +.. topic:: References: + + .. [BKNS2000] Breunig, Kriegel, Ng, and Sander (2000) + `LOF: identifying density-based local outliers. + `_ + Proc. ACM SIGMOD + +One-class SVM versus Elliptic Envelope versus Isolation Forest versus LOF +------------------------------------------------------------------------- Strictly-speaking, the One-class SVM is not an outlier-detection method, but a novelty-detection method: its training set should not be @@ -188,7 +246,8 @@ results in these situations. The examples below illustrate how the performance of the :class:`covariance.EllipticEnvelope` degrades as the data is less and less unimodal. The :class:`svm.OneClassSVM` works better on data with -multiple modes and :class:`ensemble.IsolationForest` performs well in every cases. +multiple modes and :class:`ensemble.IsolationForest` and +:class:`neighbors.LocalOutlierFactor` perform well in every cases. .. |outlier1| image:: ../auto_examples/covariance/images/sphx_glr_plot_outlier_detection_001.png :target: ../auto_examples/covariance/plot_outlier_detection.html @@ -202,7 +261,7 @@ multiple modes and :class:`ensemble.IsolationForest` performs well in every case :target: ../auto_examples/covariance/plot_outlier_detection.html :scale: 50% -.. list-table:: **Comparing One-class SVM approach, and elliptic envelope** +.. list-table:: **Comparing One-class SVM, Isolation Forest, LOF, and Elliptic Envelope** :widths: 40 60 * @@ -213,16 +272,17 @@ multiple modes and :class:`ensemble.IsolationForest` performs well in every case opposite, the decision rule based on fitting an :class:`covariance.EllipticEnvelope` learns an ellipse, which fits well the inlier distribution. The :class:`ensemble.IsolationForest` - performs as well. - - |outlier1| + and :class:`neighbors.LocalOutlierFactor` perform as well. + - |outlier1| * - As the inlier distribution becomes bimodal, the :class:`covariance.EllipticEnvelope` does not fit well the - inliers. However, we can see that both :class:`ensemble.IsolationForest` - and :class:`svm.OneClassSVM` have difficulties to detect the two modes, + inliers. However, we can see that :class:`ensemble.IsolationForest`, + :class:`svm.OneClassSVM` and :class:`neighbors.LocalOutlierFactor` + have difficulties to detect the two modes, and that the :class:`svm.OneClassSVM` - tends to overfit: because it has not model of inliers, it + tends to overfit: because it has no model of inliers, it interprets a region where, by chance some outliers are clustered, as inliers. - |outlier2| @@ -230,7 +290,8 @@ multiple modes and :class:`ensemble.IsolationForest` performs well in every case * - If the inlier distribution is strongly non Gaussian, the :class:`svm.OneClassSVM` is able to recover a reasonable - approximation as well as :class:`ensemble.IsolationForest`, + approximation as well as :class:`ensemble.IsolationForest` + and :class:`neighbors.LocalOutlierFactor`, whereas the :class:`covariance.EllipticEnvelope` completely fails. - |outlier3| @@ -238,6 +299,6 @@ multiple modes and :class:`ensemble.IsolationForest` performs well in every case * See :ref:`sphx_glr_auto_examples_covariance_plot_outlier_detection.py` for a comparison of the :class:`svm.OneClassSVM` (tuned to perform like - an outlier detection method), the :class:`ensemble.IsolationForest` - and a covariance-based outlier - detection with :class:`covariance.MinCovDet`. + an outlier detection method), the :class:`ensemble.IsolationForest`, + the :class:`neighbors.LocalOutlierFactor` + and a covariance-based outlier detection :class:`covariance.EllipticEnvelope`. diff --git a/doc/sphinxext/sphinx_issues.py b/doc/sphinxext/sphinx_issues.py new file mode 100644 index 0000000000000..27d714bf0b76b --- /dev/null +++ b/doc/sphinxext/sphinx_issues.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +"""A Sphinx extension for linking to your project's issue tracker.""" +""" +Copyright 2014 Steven Loria + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +try: + from docutils import nodes, utils +except ImportError: + # Load lazily so that test-sphinxext does not require docutils dependency + pass + +__version__ = '0.2.0' +__author__ = 'Steven Loria' +__license__ = 'MIT' + + +def user_role(name, rawtext, text, lineno, + inliner, options=None, content=None): + """Sphinx role for linking to a user profile. Defaults to linking to + Github profiles, but the profile URIS can be configured via the + ``issues_user_uri`` config value. + + Example: :: + + :user:`sloria` + """ + options = options or {} + content = content or [] + username = utils.unescape(text).strip() + config = inliner.document.settings.env.app.config + if config.issues_user_uri: + ref = config.issues_user_uri.format(user=username) + else: + ref = 'https://github.com/{0}'.format(username) + text = '@{0}'.format(username) + link = nodes.reference(text=text, refuri=ref, **options) + return [link], [] + + +def _make_issue_node(issue_no, config, options=None): + options = options or {} + if issue_no not in ('-', '0'): + if config.issues_uri: + ref = config.issues_uri.format(issue=issue_no) + elif config.issues_github_path: + ref = 'https://github.com/{0}/issues/{1}'.format( + config.issues_github_path, issue_no + ) + issue_text = '#{0}'.format(issue_no) + link = nodes.reference(text=issue_text, refuri=ref, **options) + else: + link = None + return link + + +def issue_role(name, rawtext, text, lineno, + inliner, options=None, content=None): + """Sphinx role for linking to an issue. Must have + `issues_uri` or `issues_github_path` configured in ``conf.py``. + + Examples: :: + + :issue:`123` + :issue:`42,45` + """ + options = options or {} + content = content or [] + issue_nos = [each.strip() for each in utils.unescape(text).split(',')] + config = inliner.document.settings.env.app.config + ret = [] + for i, issue_no in enumerate(issue_nos): + node = _make_issue_node(issue_no, config, options=options) + ret.append(node) + if i != len(issue_nos) - 1: + sep = nodes.raw(text=', ', format='html') + ret.append(sep) + return ret, [] + + +def setup(app): + # Format template for issues URI + # e.g. 'https://github.com/sloria/marshmallow/issues/{issue} + app.add_config_value('issues_uri', default=None, rebuild='html') + # Shortcut for Github, e.g. 'sloria/marshmallow' + app.add_config_value('issues_github_path', default=None, rebuild='html') + # Format template for user profile URI + # e.g. 'https://github.com/{user}' + app.add_config_value('issues_user_uri', default=None, rebuild='html') + app.add_role('issue', issue_role) + app.add_role('user', user_role) diff --git a/doc/templates/deprecated_class.rst b/doc/templates/deprecated_class.rst new file mode 100644 index 0000000000000..857e2c28ce1da --- /dev/null +++ b/doc/templates/deprecated_class.rst @@ -0,0 +1,23 @@ +:mod:`{{module}}`.{{objname}} +{{ underline }}============== + +.. meta:: + :robots: noindex + +.. warning:: + **DEPRECATED** + + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + + {% block methods %} + .. automethod:: __init__ + {% endblock %} + +.. include:: {{module}}.{{objname}}.examples + +.. raw:: html + +
diff --git a/doc/templates/deprecated_class_with_call.rst b/doc/templates/deprecated_class_with_call.rst new file mode 100644 index 0000000000000..a04efcb80be07 --- /dev/null +++ b/doc/templates/deprecated_class_with_call.rst @@ -0,0 +1,24 @@ +:mod:`{{module}}`.{{objname}} +{{ underline }}=============== + +.. meta:: + :robots: noindex + +.. warning:: + **DEPRECATED** + + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + + {% block methods %} + .. automethod:: __init__ + .. automethod:: __call__ + {% endblock %} + +.. include:: {{module}}.{{objname}}.examples + +.. raw:: html + +
diff --git a/doc/templates/deprecated_class_without_init.rst b/doc/templates/deprecated_class_without_init.rst new file mode 100644 index 0000000000000..c019992493610 --- /dev/null +++ b/doc/templates/deprecated_class_without_init.rst @@ -0,0 +1,19 @@ +:mod:`{{module}}`.{{objname}} +{{ underline }}============== + +.. meta:: + :robots: noindex + +.. warning:: + **DEPRECATED** + + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + +.. include:: {{module}}.{{objname}}.examples + +.. raw:: html + +
diff --git a/doc/templates/deprecated_function.rst b/doc/templates/deprecated_function.rst new file mode 100644 index 0000000000000..6d13ac6aca2de --- /dev/null +++ b/doc/templates/deprecated_function.rst @@ -0,0 +1,19 @@ +:mod:`{{module}}`.{{objname}} +{{ underline }}==================== + +.. meta:: + :robots: noindex + +.. warning:: + **DEPRECATED** + + +.. currentmodule:: {{ module }} + +.. autofunction:: {{ objname }} + +.. include:: {{module}}.{{objname}}.examples + +.. raw:: html + +
diff --git a/doc/templates/generate_deprecated.sh b/doc/templates/generate_deprecated.sh new file mode 100755 index 0000000000000..a7301fb5dc419 --- /dev/null +++ b/doc/templates/generate_deprecated.sh @@ -0,0 +1,8 @@ +#!/bin/bash +for f in [^d]*; do (head -n2 < $f; echo ' +.. meta:: + :robots: noindex + +.. warning:: + **DEPRECATED** +'; tail -n+3 $f) > deprecated_$f; done diff --git a/doc/themes/scikit-learn/layout.html b/doc/themes/scikit-learn/layout.html index 32d40e2291a01..b2c053f6eaf45 100644 --- a/doc/themes/scikit-learn/layout.html +++ b/doc/themes/scikit-learn/layout.html @@ -217,7 +217,7 @@

Machine Learning in Python

- {%- if rellinks[1:] %} + {%- if rellinks %} {%- if parents %}
@@ -225,10 +225,7 @@

Machine Learning in Python

{% endif %} - - - {%- for rellink in rellinks[1:]|reverse %} + {%- for rellink in rellinks|reverse %}
{{ rellink[3]|capitalize }} @@ -326,7 +323,7 @@

Machine Learning in Python

{% else %}
{% endif %} - {%- for rellink in rellinks[1:]|reverse %} + {%- for rellink in rellinks|reverse %}
{{ loop.cycle('Previous', 'Next') }} diff --git a/doc/themes/scikit-learn/static/js/copybutton.js b/doc/themes/scikit-learn/static/js/copybutton.js index 0a479d21ebcda..b56d9b2f0051c 100644 --- a/doc/themes/scikit-learn/static/js/copybutton.js +++ b/doc/themes/scikit-learn/static/js/copybutton.js @@ -1,12 +1,11 @@ $(document).ready(function() { /* Add a [>>>] button on the top-right corner of code samples to hide * the >>> and ... prompts and the output and thus make the code - * copyable. - * Note: This JS snippet was taken from the official python.org - * documentation site.*/ + * copyable. */ var div = $('.highlight-python .highlight,' + '.highlight-python3 .highlight,' + - '.highlight-pycon .highlight') + '.highlight-pycon .highlight,' + + '.highlight-default .highlight') var pre = div.find('pre'); // get the styles from the current theme @@ -20,7 +19,8 @@ $(document).ready(function() { 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', 'border-color': border_color, 'border-style': border_style, 'border-width': border_width, 'color': border_color, 'text-size': '75%', - 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em' + 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', + 'border-radius': '0 3px 0 0' } // create and add the button to all the code blocks that contain >>> @@ -30,6 +30,7 @@ $(document).ready(function() { var button = $('>>>'); button.css(button_styles) button.attr('title', hide_text); + button.data('hidden', 'false'); jthis.prepend(button); } // tracebacks (.gt) contain bare text elements that need to be @@ -40,20 +41,24 @@ $(document).ready(function() { }); // define the behavior of the button when it's clicked - $('.copybutton').toggle( - function() { - var button = $(this); + $('.copybutton').click(function(e){ + e.preventDefault(); + var button = $(this); + if (button.data('hidden') === 'false') { + // hide the code output button.parent().find('.go, .gp, .gt').hide(); button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); button.css('text-decoration', 'line-through'); button.attr('title', show_text); - }, - function() { - var button = $(this); + button.data('hidden', 'true'); + } else { + // show the code output button.parent().find('.go, .gp, .gt').show(); button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); button.css('text-decoration', 'none'); button.attr('title', hide_text); - }); + button.data('hidden', 'false'); + } + }); }); diff --git a/doc/themes/scikit-learn/static/nature.css_t b/doc/themes/scikit-learn/static/nature.css_t index 86449d22204f3..593cb01ce67c6 100644 --- a/doc/themes/scikit-learn/static/nature.css_t +++ b/doc/themes/scikit-learn/static/nature.css_t @@ -551,6 +551,30 @@ div.highlight a { text-decoration: underline; } +div.highlight:hover span.copybutton { + background-color: #3F556B; +} + +div.highlight:hover span.copybutton:hover { + background-color: #20252B; +} + +@media (min-width: 1060px) { + div.highlight:hover span.copybutton:after{ + background: #3F556B; + border-radius: 5px; + color: white; + content: attr(title); + left: 110%; + padding: 5px 15px; + position: absolute; + z-index: 98; + width: 140px; + top: -10px; + } +} + + div.note { background-color: #eee; border: 1px solid #ccc; @@ -579,7 +603,7 @@ div.admonition { -moz-border-radius: 4px; } -div.warning { +div.warning, div.deprecated { color: #b94a48; background-color: #F3E5E5; border: 1px solid #eed3d7; diff --git a/doc/whats_new.rst b/doc/whats_new.rst index b5f10da91d28f..0f049ed40005f 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -16,45 +16,68 @@ Changelog New features ............ + - Added the :class:`neighbors.LocalOutlierFactor` class for anomaly detection based + on nearest neighbors. By `Nicolas Goix`_ and `Alexandre Gramfort`_. + Enhancements ............ + - Added ``classes_`` attribute to :class:`model_selection.GridSearchCV` + that matches the ``classes_`` attribute of ``best_estimator_``. (`#7661 + `_) by `Alyssa + Batula`_ and `Dylan Werner-Meier`_. + - The ``min_weight_fraction_leaf`` constraint in tree construction is now more efficient, taking a fast path to declare a node a leaf if its weight is less than 2 * the minimum. Note that the constructed tree will be different from previous versions where ``min_weight_fraction_leaf`` is - used. (`#7441 `_) - by `Nelson Liu`_. + used. :issue:`7441` by `Nelson Liu`_. - Added ``average`` parameter to perform weights averaging in - :class:`linear_model.PassiveAggressiveClassifier`. (`#4939 - `_) by `Andrea - Esuli`_. + :class:`linear_model.PassiveAggressiveClassifier`. :issue:`4939` + by :user:`Andrea Esuli `. - Custom metrics for the :mod:`sklearn.neighbors` binary trees now have fewer constraints: they must take two 1d-arrays and return a float. - (`#6288 `_) by - `Jake VanderPlas`_. + :issue:`6288` by `Jake VanderPlas`_. + + - :class:`ensemble.GradientBoostingClassifier` and :class:`ensemble.GradientBoostingRegressor` + now support sparse input for prediction. + :issue:`6101` by `Ibraim Ganiev`_. + + - Added ``shuffle`` and ``random_state`` parameters to shuffle training + data before taking prefixes of it based on training sizes in + :func:`model_selection.learning_curve`. + (`#7506` _) by + `Narine Kokhlikyan`_. + + - Added ``norm_order`` parameter to :class:`feature_selection.SelectFromModel` + to enable selection of the norm order when ``coef_`` is more than 1D Bug fixes ......... + - Fix a bug where :class:`sklearn.feature_selection.SelectFdr` did not + exactly implement Benjamini-Hochberg procedure. It formerly may have + selected fewer features than it should. + (`#7490 `_) by + `Peng Meng`_. + - :class:`sklearn.manifold.LocallyLinearEmbedding` now correctly handles - integer inputs - (`#6282 `_) by - `Jake Vanderplas`_. + integer inputs. :issue:`6282` by `Jake Vanderplas`_. - The ``min_weight_fraction_leaf`` parameter of tree-based classifiers and regressors now assumes uniform sample weights by default if the ``sample_weight`` argument is not passed to the ``fit`` function. - Previously, the parameter was silently ignored. (`#7301 - `_) by `Nelson - Liu`_. + Previously, the parameter was silently ignored. :issue:`7301` + by `Nelson Liu`_. - Numerical issue with :class:`linear_model.RidgeCV` on centered data when - `n_features > n_samples`. (`#6178 - `_) by `Bertrand - Thirion`_ + `n_features > n_samples`. :issue:`6178` by `Bertrand Thirion`_ + + - Tree splitting criterion classes' cloning/pickling is now memory safe + (`#7680 `_). + By `Ibraim Ganiev`_. .. _changes_0_18_1: @@ -66,8 +89,31 @@ Bug fixes - Fix issue where ``min_grad_norm`` and ``n_iter_without_progress`` parameters were not being utilised by :class:`manifold.TSNE`. - `#6497 `_ - by `Sebastian Säger`_ + :issue:`6497` by `Sebastian Säger`_ + + - Attribute ``explained_variance_ratio`` of + :class:`discriminant_analysis.LinearDiscriminantAnalysis` calculated + with SVD and Eigen solver are now of the same length. (`#7632 + `_). + By `JPFrancoia`_ + + - Fixes issue in :ref:`univariate_feature_selection` where score + functions were not accepting multi-label targets.(`#7676 + `_) + by `Mohammed Affan`_ + + +API changes summary +------------------- + +Linear, kernelized and related models + + - Length of `explained_variance_ratio` of + :class:`discriminant_analysis.LinearDiscriminantAnalysis` + changed for both Eigen and SVD solvers. The attribute has now a length + of min(n_components, n_classes - 1). (`#7632 + `_). + By `JPFrancoia`_ .. _changes_0_18: @@ -199,10 +245,10 @@ Classifiers and Regressors examples are provided. By `Jan Hendrik Metzen`_. - Added new supervised learning algorithm: :ref:`Multi-layer Perceptron ` - (`#3204 `_) by `Issam H. Laradji`_ + :issue:`3204` by `Issam H. Laradji`_ - Added :class:`linear_model.HuberRegressor`, a linear model robust to outliers. - (`#5291 `_) by `Manoj Kumar`_. + :issue:`5291` by `Manoj Kumar`_. - Added the :class:`multioutput.MultiOutputRegressor` meta-estimator. It converts single output regressors to multi-ouput regressors by fitting @@ -212,9 +258,7 @@ Other estimators - New :class:`mixture.GaussianMixture` and :class:`mixture.BayesianGaussianMixture` replace former mixture models, employing faster inference - for sounder results. - (`#7295 `_) by - `Wei Xue`_ and `Thierry Guillemot`_. + for sounder results. :issue:`7295` by `Wei Xue`_ and `Thierry Guillemot`_. - Class :class:`decomposition.RandomizedPCA` is now factored into :class:`decomposition.PCA` and it is available calling with parameter ``svd_solver='randomized'``. @@ -222,8 +266,7 @@ Other estimators behavior of PCA is recovered by ``svd_solver='full'``. An additional solver calls ``arpack`` and performs truncated (non-randomized) SVD. By default, the best solver is selected depending on the size of the input and the - number of components requested. - (`#5299 `_) by `Giorgio Patrini`_. + number of components requested. :issue:`5299` by `Giorgio Patrini`_. - Added two functions for mutual information estimation: :func:`feature_selection.mutual_info_classif` and @@ -250,16 +293,12 @@ Model selection and evaluation - Added new cross-validation splitter :class:`model_selection.TimeSeriesSplit` to handle time series data. - (`#6586 - `_) by `YenChen - Lin`_ + :issue:`6586` by `YenChen Lin`_ - The cross-validation iterators are replaced by cross-validation splitters available from :mod:`sklearn.model_selection`, allowing for nested - cross-validation. - See :ref:`model_selection_changes` for more information. - (`#4294 `_) by - `Raghav R V`_. + cross-validation. See :ref:`model_selection_changes` for more information. + :issue:`4294` by `Raghav R V`_. Enhancements ............ @@ -270,14 +309,10 @@ Trees and ensembles the mean absolute error. This criterion can also be used in :class:`ensemble.ExtraTreesRegressor`, :class:`ensemble.RandomForestRegressor`, and the gradient boosting - estimators. (`#6667 - `_) by `Nelson - Liu`_. + estimators. :issue:`6667` by `Nelson Liu`_. - Added weighted impurity-based early stopping criterion for decision tree - growth. (`#6954 - `_) by `Nelson - Liu`_ + growth. :issue:`6954` by `Nelson Liu`_ - The random forest, extra tree and decision tree estimators now has a method ``decision_path`` which returns the decision path of samples in @@ -288,13 +323,11 @@ Trees and ensembles - Random forest, extra trees, decision trees and gradient boosting estimator accept the parameter ``min_samples_split`` and ``min_samples_leaf`` - provided as a percentage of the training samples. By - `yelite`_ and `Arnaud Joly`_. + provided as a percentage of the training samples. By `yelite`_ and `Arnaud Joly`_. - Gradient boosting estimators accept the parameter ``criterion`` to specify - to splitting criterion used in built decision trees. (`#6667 - `_) by `Nelson - Liu`_. + to splitting criterion used in built decision trees. + :issue:`6667` by `Nelson Liu`_. - The memory footprint is reduced (sometimes greatly) for :class:`ensemble.bagging.BaseBagging` and classes that inherit from it, @@ -305,15 +338,12 @@ Trees and ensembles - Added ``n_jobs`` and ``sample_weights`` parameters for :class:`ensemble.VotingClassifier` to fit underlying estimators in parallel. - (`#5805 `_) - By `Ibraim Ganiev`_. + :issue:`5805` by `Ibraim Ganiev`_. Linear, kernelized and related models - In :class:`linear_model.LogisticRegression`, the SAG solver is now - available in the multinomial case. - (`#5251 `_) - By `Tom Dupre la Tour`_. + available in the multinomial case. :issue:`5251` by `Tom Dupre la Tour`_. - :class:`linear_model.RANSACRegressor`, :class:`svm.LinearSVC` and :class:`svm.LinearSVR` now support ``sample_weights``. @@ -328,8 +358,7 @@ Linear, kernelized and related models - Isotonic regression (:class:`isotonic.IsotonicRegression`) now uses a better algorithm to avoid `O(n^2)` behavior in pathological cases, and is also generally faster - (`#6601 `_). - By `Antony Lee`_. + (:issue:`#6691`). By `Antony Lee`_. - :class:`naive_bayes.GaussianNB` now accepts data-independent class-priors through the parameter ``priors``. By `Guillaume Lemaitre`_. @@ -337,13 +366,11 @@ Linear, kernelized and related models - :class:`linear_model.ElasticNet` and :class:`linear_model.Lasso` now works with ``np.float32`` input data without converting it into ``np.float64``. This allows to reduce the memory - consumption. - (`#6913 `_) - By `YenChen Lin`_. + consumption. :issue:`6913` by `YenChen Lin`_. - :class:`semi_supervised.LabelPropagation` and :class:`semi_supervised.LabelSpreading` now accept arbitrary kernel functions in addition to strings ``knn`` and ``rbf``. - (`#5762 `_) By `Utkarsh Upadhyay`_. + :issue:`5762` by `Utkarsh Upadhyay`_. Decomposition, manifold learning and clustering @@ -353,18 +380,15 @@ Decomposition, manifold learning and clustering - :class:`cluster.KMeans` and :class:`cluster.MiniBatchKMeans` now works with ``np.float32`` and ``np.float64`` input data without converting it. This allows to reduce the memory consumption by using ``np.float32``. - (`#6846 `_) - By `Sebastian Säger`_ and `YenChen Lin`_. + :issue:`6846` by `Sebastian Säger`_ and `YenChen Lin`_. Preprocessing and feature selection - :class:`preprocessing.RobustScaler` now accepts ``quantile_range`` parameter. - (`#5929 `_) - By `Konstantin Podshumok`_. + :issue:`5929` by `Konstantin Podshumok`_. - :class:`feature_extraction.FeatureHasher` now accepts string values. - (`#6173 `_) By `Ryad Zenine`_ - and `Devashish Deshpande`_. + :issue:`6173` by `Ryad Zenine`_ and `Devashish Deshpande`_. - Keyword arguments can now be supplied to ``func`` in :class:`preprocessing.FunctionTransformer` by means of the ``kw_args`` @@ -388,9 +412,7 @@ Model evaluation and meta-estimators - The new ``cv_results_`` attribute of :class:`model_selection.GridSearchCV` (and :class:`model_selection.RandomizedSearchCV`) can be easily imported into pandas as a ``DataFrame``. Ref :ref:`model_selection_changes` for - more information. - (`#6697 `_) by - `Raghav R V`_. + more information. :issue:`6697` by `Raghav R V`_. - Generalization of :func:`model_selection.cross_val_predict`. One can pass method names such as `predict_proba` to be used in the cross @@ -400,21 +422,18 @@ Model evaluation and meta-estimators - The training scores and time taken for training followed by scoring for each search candidate are now available at the ``cv_results_`` dict. See :ref:`model_selection_changes` for more information. - (`#7324 `_) - By `Eugene Chen`_ and `Raghav R V`_. + :issue:`7325` by `Eugene Chen`_ and `Raghav R V`_. Metrics - Added ``labels`` flag to :class:`metrics.log_loss` to to explicitly provide the labels when the number of classes in ``y_true`` and ``y_pred`` differ. - (`#7239 `_) - by `Hong Guangguo`_ with help from `Mads Jensen`_ and `Nelson Liu`_. + :issue:`7239` by `Hong Guangguo`_ with help from `Mads Jensen`_ and `Nelson Liu`_. - Support sparse contingency matrices in cluster evaluation (:mod:`metrics.cluster.supervised`) to scale to a large number of clusters. - (`#7419 `_) - By `Gregory Stupp`_ and `Joel Nothman`_. + :issue:`7419` by `Gregory Stupp`_ and `Joel Nothman`_. - Add ``sample_weight`` parameter to :func:`metrics.matthews_corrcoef`. By `Jatin Shah`_ and `Raghav R V`_. @@ -444,26 +463,22 @@ Miscellaneous - Added parameter ``return_X_y`` and return type ``(data, target) : tuple`` option to :func:`load_iris` dataset - `#7049 `_, + :issue:`7049`, :func:`load_breast_cancer` dataset - `#7152 `_, + :issue:`7152`, :func:`load_digits` dataset, :func:`load_diabetes` dataset, :func:`load_linnerud` dataset, :func:`load_boston` dataset - `#7154 `_ by + :issue:`7154` by `Manvendra Singh`_. - Simplification of the ``clone`` function, deprecate support for estimators - that modify parameters in ``__init__``. - (`#5540 `_) - By `Andreas Müller`_. + that modify parameters in ``__init__``. :issue:`5540` by `Andreas Müller`_. - When unpickling a scikit-learn estimator in a different version than the one the estimator was trained with, a ``UserWarning`` is raised, see :ref:`the documentation - on model persistence ` - for more details. - (`#7248 `_) + on model persistence ` for more details. (:issue:`7248`) By `Andreas Müller`_. Bug fixes @@ -482,13 +497,11 @@ Trees and ensembles - Fix bug where :class:`ensemble.AdaBoostClassifier` and :class:`ensemble.AdaBoostRegressor` would perform poorly if the ``random_state`` was fixed - (`#7411 `_). - By `Joel Nothman`_. + (:issue:`7411`). By `Joel Nothman`_. - Fix bug in ensembles with randomization where the ensemble would not set ``random_state`` on base estimators in a pipeline or similar nesting. - (`#7411 `_). - Note, results for :class:`ensemble.BaggingClassifier` + (:issue:`7411`). Note, results for :class:`ensemble.BaggingClassifier` :class:`ensemble.BaggingRegressor`, :class:`ensemble.AdaBoostClassifier` and :class:`ensemble.AdaBoostRegressor` will now differ from previous versions. By `Joel Nothman`_. @@ -497,17 +510,15 @@ Linear, kernelized and related models - Fixed incorrect gradient computation for ``loss='squared_epsilon_insensitive'`` in :class:`linear_model.SGDClassifier` and :class:`linear_model.SGDRegressor` - (`#6764 `_). By `Wenhua Yang`_. + (:issue:`6764`). By `Wenhua Yang`_. - Fix bug in :class:`linear_model.LogisticRegressionCV` where ``solver='liblinear'`` did not accept ``class_weights='balanced``. - (`#6817 `_). - By `Tom Dupre la Tour`_. + (:issue:`6817`). By `Tom Dupre la Tour`_. - Fix bug in :class:`neighbors.RadiusNeighborsClassifier` where an error occurred when there were outliers being labelled and a weight function - specified (`#6902 - `_). By + specified (:issue:`6902`). By `LeonieBorne `_. - Fix :class:`linear_model.ElasticNet` sparse decision function to match @@ -516,64 +527,60 @@ Linear, kernelized and related models Decomposition, manifold learning and clustering - :class:`decomposition.RandomizedPCA` default number of `iterated_power` is 4 instead of 3. - (`#5141 `_) by `Giorgio Patrini`_. + :issue:`5141` by `Giorgio Patrini`_. - :func:`utils.extmath.randomized_svd` performs 4 power iterations by default, instead or 0. In practice this is enough for obtaining a good approximation of the true eigenvalues/vectors in the presence of noise. When `n_components` is small (``< .1 * min(X.shape)``) `n_iter` is set to 7, unless the user specifies a higher number. This improves precision with few components. - (`#5299 `_) by `Giorgio Patrini`_. + :issue:`5299` by `Giorgio Patrini`_. - Whiten/non-whiten inconsistency between components of :class:`decomposition.PCA` and :class:`decomposition.RandomizedPCA` (now factored into PCA, see the New features) is fixed. `components_` are stored with no whitening. - (`#5299 `_) by `Giorgio Patrini`_. + :issue:`5299` by `Giorgio Patrini`_. - Fixed bug in :func:`manifold.spectral_embedding` where diagonal of unnormalized - Laplacian matrix was incorrectly set to 1. (`#4995 `_) By `Peter Fischer`_. + Laplacian matrix was incorrectly set to 1. :issue:`4995` by `Peter Fischer`_. - Fixed incorrect initialization of :func:`utils.arpack.eigsh` on all occurrences. Affects :class:`cluster.bicluster.SpectralBiclustering`, :class:`decomposition.KernelPCA`, :class:`manifold.LocallyLinearEmbedding`, - and :class:`manifold.SpectralEmbedding` (`#5012 `_). By `Peter Fischer`_. + and :class:`manifold.SpectralEmbedding` (:issue:`5012`). By `Peter Fischer`_. - - Attribute ``explained_variance_ratio_`` calculated with the SVD solver of - :class:`discriminant_analysis.LinearDiscriminantAnalysis` now returns + - Attribute ``explained_variance_ratio_`` calculated with the SVD solver + of :class:`discriminant_analysis.LinearDiscriminantAnalysis` now returns correct results. By `JPFrancoia`_ Preprocessing and feature selection - :func:`preprocessing.data._transform_selected` now always passes a copy - of ``X`` to transform function when ``copy=True`` (`#7194 - `_). By `Caio + of ``X`` to transform function when ``copy=True`` (:issue:`7194`). By `Caio Oliveira `_. Model evaluation and meta-estimators - :class:`model_selection.StratifiedKFold` now raises error if all n_labels for individual classes is less than n_folds. - (`#6182 `_) by `Devashish Deshpande`_. + :issue:`6182` by `Devashish Deshpande`_. - Fixed bug in :class:`model_selection.StratifiedShuffleSplit` where train and test sample could overlap in some edge cases, - see `#6121 `_ for + see :issue:`6121` for more details. By `Loic Esteve`_. - Fix in :class:`sklearn.model_selection.StratifiedShuffleSplit` to return splits of size ``train_size`` and ``test_size`` in all cases - (`#6472 `_). - By `Andreas Müller`_. + (:issue:`6472`). By `Andreas Müller`_. - Cross-validation of :class:`OneVsOneClassifier` and :class:`OneVsRestClassifier` now works with precomputed kernels. - (`#7350 `_) - By `Russell Smith`_. + :issue:`7350` by `Russell Smith`_. - Fix incomplete ``predict_proba`` method delegation from :class:`model_selection.GridSearchCV` to - :class:`linear_model.SGDClassifier` (`#7159 - `_) + :class:`linear_model.SGDClassifier` (:issue:`7159`) by `Yichuan Liu `_. Metrics @@ -590,37 +597,33 @@ Metrics - :func:`metrics.pairwise.pairwise_distances` now converts arrays to boolean arrays when required in ``scipy.spatial.distance``. - (`#5460 `_) - By `Tom Dupre la Tour`_. + :issue:`5460` by `Tom Dupre la Tour`_. - Fix sparse input support in :func:`metrics.silhouette_score` as well as example examples/text/document_clustering.py. By `YenChen Lin`_. - :func:`metrics.roc_curve` and :func:`metrics.precision_recall_curve` no longer round ``y_score`` values when creating ROC curves; this was causing - problems for users with very small differences in scores (`#7353 - `_). + problems for users with very small differences in scores (:issue:`7353`). Miscellaneous - :func:`model_selection.tests._search._check_param_grid` now works correctly with all types that extends/implements `Sequence` (except string), including range (Python 3.x) and xrange - (Python 2.x). - (`#7323 `_) by Viacheslav Kovalevskyi. + (Python 2.x). :issue:`7323` by Viacheslav Kovalevskyi. - :func:`utils.extmath.randomized_range_finder` is more numerically stable when many power iterations are requested, since it applies LU normalization by default. If ``n_iter<2`` numerical issues are unlikely, thus no normalization is applied. Other normalization options are available: ``'none', 'LU'`` and ``'QR'``. - (`#5141 `_) by `Giorgio Patrini`_. + :issue:`5141` by `Giorgio Patrini`_. - Fix a bug where some formats of ``scipy.sparse`` matrix, and estimators with them as parameters, could not be passed to :func:`base.clone`. By `Loic Esteve`_. - :func:`datasets.load_svmlight_file` now is able to read long int QID values. - (`#7101 `_) - By `Ibraim Ganiev`_. + :issue:`7101` by `Ibraim Ganiev`_. API changes summary @@ -642,8 +645,7 @@ Decomposition, manifold learning and clustering The new class solves the computational problems of the old class and computes the Gaussian mixture with a Dirichlet process prior faster than before. - (`#7295 `_) by - `Wei Xue`_ and `Thierry Guillemot`_. + :issue:`7295` by `Wei Xue`_ and `Thierry Guillemot`_. - The old :class:`mixture.VBGMM` is deprecated in favor of the new :class:`mixture.BayesianGaussianMixture` (with the parameter @@ -651,14 +653,12 @@ Decomposition, manifold learning and clustering The new class solves the computational problems of the old class and computes the Variational Bayesian Gaussian mixture faster than before. - (`#6651 `_) by - `Wei Xue`_ and `Thierry Guillemot`_. + :issue:`6651` by `Wei Xue`_ and `Thierry Guillemot`_. - The old :class:`mixture.GMM` is deprecated in favor of the new :class:`mixture.GaussianMixture`. The new class computes the Gaussian mixture faster than before and some of computational problems have been solved. - (`#6666 `_) by - `Wei Xue`_ and `Thierry Guillemot`_. + :issue:`6666` by `Wei Xue`_ and `Thierry Guillemot`_. Model evaluation and meta-estimators @@ -666,26 +666,21 @@ Model evaluation and meta-estimators :mod:`sklearn.learning_curve` have been deprecated and the classes and functions have been reorganized into the :mod:`sklearn.model_selection` module. Ref :ref:`model_selection_changes` for more information. - (`#4294 `_) by - `Raghav R V`_. + :issue:`4294` by `Raghav R V`_. - The ``grid_scores_`` attribute of :class:`model_selection.GridSearchCV` and :class:`model_selection.RandomizedSearchCV` is deprecated in favor of the attribute ``cv_results_``. Ref :ref:`model_selection_changes` for more information. - (`#6697 `_) by - `Raghav R V`_. + :issue:`6697` by `Raghav R V`_. - The parameters ``n_iter`` or ``n_folds`` in old CV splitters are replaced by the new parameter ``n_splits`` since it can provide a consistent and unambiguous interface to represent the number of train-test splits. - (`#7187 `_) - by `YenChen Lin`_. + :issue:`7187` by `YenChen Lin`_. - ``classes`` parameter was renamed to ``labels`` in - :func:`metrics.hamming_loss`. - (`#7260 `_) by - `Sebastián Vanrell`_. + :func:`metrics.hamming_loss`. :issue:`7260` by `Sebastián Vanrell`_. - The splitter classes ``LabelKFold``, ``LabelShuffleSplit``, ``LeaveOneLabelOut`` and ``LeavePLabelsOut`` are renamed to @@ -698,8 +693,7 @@ Model evaluation and meta-estimators :class:`model_selection.LeavePGroupsOut` is renamed to ``groups``. Additionally in :class:`model_selection.LeavePGroupsOut`, the parameter ``n_labels`` is renamed to ``n_groups``. - (`#6660 `_) - by `Raghav R V`_. + :issue:`6660` by `Raghav R V`_. Code Contributors ----------------- @@ -781,24 +775,20 @@ Bug fixes - Fixed reading of Bunch pickles generated with scikit-learn version <= 0.16. This can affect users who have already downloaded a dataset with scikit-learn 0.16 and are loading it - with scikit-learn 0.17. See `#6196 - `_ for + with scikit-learn 0.17. See :issue:`6196` for how this affected :func:`datasets.fetch_20newsgroups`. By `Loic Esteve`_. - Fixed a bug that prevented using ROC AUC score to perform grid search on - several CPU / cores on large arrays. See `#6147 - `_ + several CPU / cores on large arrays. See :issue:`6147` By `Olivier Grisel`_. - Fixed a bug that prevented to properly set the ``presort`` parameter - in :class:`ensemble.GradientBoostingRegressor`. See `#5857 - `_ + in :class:`ensemble.GradientBoostingRegressor`. See :issue:`5857` By Andrew McCulloh. - Fixed a joblib error when evaluating the perplexity of a - :class:`decomposition.LatentDirichletAllocation` model. See `#6258 - `_ + :class:`decomposition.LatentDirichletAllocation` model. See :issue:`6258` By Chyi-Kwei Yau. .. _changes_0_17: @@ -845,13 +835,13 @@ New features - :class:`decomposition.LatentDirichletAllocation` implements the Latent Dirichlet Allocation topic model with online variational inference. By `Chyi-Kwei Yau`_, with code based on an implementation - by Matt Hoffman. (`#3659 `_) + by Matt Hoffman. (:issue:`3659`) - The new solver ``sag`` implements a Stochastic Average Gradient descent and is available in both :class:`linear_model.LogisticRegression` and :class:`linear_model.Ridge`. This solver is very efficient for large datasets. By `Danny Sullivan`_ and `Tom Dupre la Tour`_. - (`#4738 `_) + (:issue:`4738`) - The new solver ``cd`` implements a Coordinate Descent in :class:`decomposition.NMF`. Previous solver based on Projected Gradient is @@ -867,7 +857,7 @@ Enhancements ............ - :class:`manifold.TSNE` now supports approximate optimization via the Barnes-Hut method, leading to much faster fitting. By Christopher Erick Moody. - (`#4025 `_) + (:issue:`4025`) - :class:`cluster.mean_shift_.MeanShift` now supports parallel execution, as implemented in the ``mean_shift`` function. By `Martino Sorbaro`_. @@ -887,8 +877,7 @@ Enhancements - Added the :func:`metrics.cohen_kappa_score` metric. - Added a ``warm_start`` constructor parameter to the bagging ensemble - models to increase the size of the ensemble. By - `Tim Head`_. + models to increase the size of the ensemble. By `Tim Head`_. - Added option to use multi-output regression metrics without averaging. By Konstantin Shmelkov and `Michael Eickenberg`_. @@ -990,11 +979,10 @@ Enhancements each try. By `Jacob Schreiber`_. - Add ``sample_weight`` support to :class:`linear_model.LinearRegression`. - By Sonny Hu. (`#4481 `_) + By Sonny Hu. (:issue:`#4881`) - Add ``n_iter_without_progress`` to :class:`manifold.TSNE` to control - the stopping criterion. By Santi Villalba. - (`#5185 `_) + the stopping criterion. By Santi Villalba. (:issue:`5186`) - Added optional parameter ``random_state`` in :class:`linear_model.Ridge` , to set the seed of the pseudo random generator used in ``sag`` solver. By `Tom Dupre la Tour`_. @@ -1035,7 +1023,7 @@ Enhancements - Added ``positive`` option to :class:`linear_model.Lars` and :func:`linear_model.lars_path` to force coefficients to be positive. - (`#5131 `) + (:issue:`5131`) - Added the ``X_norm_squared`` parameter to :func:`metrics.pairwise.euclidean_distances` to provide precomputed squared norms for ``X``. @@ -1067,7 +1055,7 @@ Bug fixes - All regressors now consistently handle and warn when given ``y`` that is of shape ``(n_samples, 1)``. By `Andreas Müller`_ and Henry Lin. - (`#5431 `_) + (:issue:`5431`) - Fix in :class:`cluster.KMeans` cluster reassignment for sparse input by `Lars Buitinck`_. @@ -1080,15 +1068,15 @@ Bug fixes - Fixed the ``predict_proba`` method of :class:`linear_model.LogisticRegression` to use soft-max instead of one-vs-rest normalization. By `Manoj Kumar`_. - (`#5182 `_) + (:issue:`5182`) - Fixed the :func:`partial_fit` method of :class:`linear_model.SGDClassifier` when called with ``average=True``. By `Andrew Lamb`_. - (`#5282 `_) + (:issue:`5282`) - Dataset fetchers use different filenames under Python 2 and Python 3 to avoid pickling compatibility issues. By `Olivier Grisel`_. - (`#5355 `_) + (:issue:`5355`) - Fixed a bug in :class:`naive_bayes.GaussianNB` which caused classification results to depend on scale. By `Jake Vanderplas`_. @@ -1096,12 +1084,10 @@ Bug fixes - Fixed temporarily :class:`linear_model.Ridge`, which was incorrect when fitting the intercept in the case of sparse data. The fix automatically changes the solver to 'sag' in this case. - (`#5360 `_) - By `Tom Dupre la Tour`_. + :issue:`5360` by `Tom Dupre la Tour`_. - Fixed a performance bug in :class:`decomposition.RandomizedPCA` on data - with a large number of features and fewer samples. (`#4478 - `_) + with a large number of features and fewer samples. (:issue:`4478`) By `Andreas Müller`_, `Loic Esteve`_ and `Giorgio Patrini`_. - Fixed bug in :class:`cross_decomposition.PLS` that yielded unstable and @@ -1123,14 +1109,14 @@ Bug fixes - Fixed inconsistent memory layout in the coordinate descent solver that affected :class:`linear_model.DictionaryLearning` and - :class:`covariance.GraphLasso`. (`#5337 `_) + :class:`covariance.GraphLasso`. (:issue:`5337`) By `Olivier Grisel`_. - :class:`manifold.LocallyLinearEmbedding` no longer ignores the ``reg`` parameter. - Nearest Neighbor estimators with custom distance metrics can now be pickled. - (`4362 `_) + (:issue:`4362`) - Fixed a bug in :class:`pipeline.FeatureUnion` where ``transformer_weights`` were not properly handled when performing grid-searches. @@ -1140,7 +1126,7 @@ Bug fixes ``class_weight='balanced'```or ``class_weight='auto'``. By `Tom Dupre la Tour`_. - - Fixed bug `#5495 `_ when + - Fixed bug :issue:`5495` when doing OVR(SVC(decision_function_shape="ovr")). Fixed by `Elvis Dohmatob`_. @@ -1785,8 +1771,8 @@ Pedregosa, Florian Wilhelm, floydsoft, Félix-Antoine Fortin, Gael Varoquaux, Garrett-R, Gilles Louppe, gpassino, gwulfs, Hampus Bengtsson, Hamzeh Alsalhi, Hanna Wallach, Harry Mavroforakis, Hasil Sharma, Helder, Herve Bredin, Hsiang-Fu Yu, Hugues SALAMIN, Ian Gilmore, Ilambharathi Kanniah, Imran Haque, -isms, Jake VanderPlas, Jan Dlabal, Jan Hendrik Metzen, Jatin Shah, Javier López -Peña, jdcaballero, Jean Kossaifi, Jeff Hammerbacher, Joel Nothman, Jonathan +isms, Jake VanderPlas, Jan Dlabal, Jan Hendrik Metzen, Jatin Shah, Javier López +Peña, jdcaballero, Jean Kossaifi, Jeff Hammerbacher, Joel Nothman, Jonathan Helmus, Joseph, Kaicheng Zhang, Kevin Markham, Kyle Beauchamp, Kyle Kastner, Lagacherie Matthieu, Lars Buitinck, Laurent Direr, leepei, Loic Esteve, Luis Pedro Coelho, Lukas Michelbacher, maheshakya, Manoj Kumar, Manuel, Mario @@ -4757,7 +4743,7 @@ David Huard, Dave Morrill, Ed Schofield, Travis Oliphant, Pearu Peterson. .. _Eric Martin: http://www.ericmart.in -.. _Nicolas Goix: https://webperso.telecom-paristech.fr/front/frontoffice.php?SP_ID=241 +.. _Nicolas Goix: https://perso.telecom-paristech.fr/~goix/ .. _Cory Lorenz: https://github.com/clorenz7 @@ -4856,3 +4842,11 @@ David Huard, Dave Morrill, Ed Schofield, Travis Oliphant, Pearu Peterson. .. _Utkarsh Upadhyay: https://github.com/musically-ut .. _Eugene Chen: https://github.com/eyc88 + +.. _Narine Kokhlikyan: https://github.com/NarineK + +.. _Peng Meng: https://github.com/mpjlu + +.. _Dylan Werner-Meier: https://github.com/unautre + +.. _Alyssa Batula: https://github.com/abatula diff --git a/examples/applications/topics_extraction_with_nmf_lda.py b/examples/applications/topics_extraction_with_nmf_lda.py index 81a0fd8457bed..d5b9cbfc5af44 100644 --- a/examples/applications/topics_extraction_with_nmf_lda.py +++ b/examples/applications/topics_extraction_with_nmf_lda.py @@ -3,8 +3,8 @@ Topic extraction with Non-negative Matrix Factorization and Latent Dirichlet Allocation ======================================================================================= -This is an example of applying Non-negative Matrix Factorization -and Latent Dirichlet Allocation on a corpus of documents and +This is an example of applying :class:`sklearn.decomposition.NMF` +and :class:`sklearn.decomposition.LatentDirichletAllocation` on a corpus of documents and extract additive models of the topic structure of the corpus. The output is a list of topics, each represented as a list of terms (weights are not shown). diff --git a/examples/covariance/plot_outlier_detection.py b/examples/covariance/plot_outlier_detection.py index ebf5a7136fa12..838906573a2ae 100644 --- a/examples/covariance/plot_outlier_detection.py +++ b/examples/covariance/plot_outlier_detection.py @@ -18,6 +18,9 @@ hence more adapted to large-dimensional settings, even if it performs quite well in the examples below. +- using the Local Outlier Factor to measure the local deviation of a given + data point with respect to its neighbors by comparing their local density. + The ground truth about inliers and outliers is given by the points colors while the orange-filled area indicates which points are reported as inliers by each method. @@ -27,7 +30,6 @@ threshold on the decision_function to separate out the corresponding fraction. """ -print(__doc__) import numpy as np from scipy import stats @@ -37,6 +39,9 @@ from sklearn import svm from sklearn.covariance import EllipticEnvelope from sklearn.ensemble import IsolationForest +from sklearn.neighbors import LocalOutlierFactor + +print(__doc__) rng = np.random.RandomState(42) @@ -52,10 +57,13 @@ "Robust covariance": EllipticEnvelope(contamination=outliers_fraction), "Isolation Forest": IsolationForest(max_samples=n_samples, contamination=outliers_fraction, - random_state=rng)} + random_state=rng), + "Local Outlier Factor": LocalOutlierFactor( + n_neighbors=35, + contamination=outliers_fraction)} # Compare given classifiers under given settings -xx, yy = np.meshgrid(np.linspace(-7, 7, 500), np.linspace(-7, 7, 500)) +xx, yy = np.meshgrid(np.linspace(-7, 7, 100), np.linspace(-7, 7, 100)) n_inliers = int((1. - outliers_fraction) * n_samples) n_outliers = int(outliers_fraction * n_samples) ground_truth = np.ones(n_samples, dtype=int) @@ -72,19 +80,27 @@ X = np.r_[X, np.random.uniform(low=-6, high=6, size=(n_outliers, 2))] # Fit the model - plt.figure(figsize=(10.8, 3.6)) + plt.figure(figsize=(9, 7)) for i, (clf_name, clf) in enumerate(classifiers.items()): # fit the data and tag outliers - clf.fit(X) - scores_pred = clf.decision_function(X) + if clf_name == "Local Outlier Factor": + y_pred = clf.fit_predict(X) + scores_pred = clf.negative_outlier_factor_ + else: + clf.fit(X) + scores_pred = clf.decision_function(X) + y_pred = clf.predict(X) threshold = stats.scoreatpercentile(scores_pred, 100 * outliers_fraction) - y_pred = clf.predict(X) n_errors = (y_pred != ground_truth).sum() # plot the levels lines and the points - Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) + if clf_name == "Local Outlier Factor": + # decision_function is private for LOF + Z = clf._decision_function(np.c_[xx.ravel(), yy.ravel()]) + else: + Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) Z = Z.reshape(xx.shape) - subplot = plt.subplot(1, 3, i + 1) + subplot = plt.subplot(2, 2, i + 1) subplot.contourf(xx, yy, Z, levels=np.linspace(Z.min(), threshold, 7), cmap=plt.cm.Blues_r) a = subplot.contour(xx, yy, Z, levels=[threshold], @@ -97,11 +113,12 @@ subplot.legend( [a.collections[0], b, c], ['learned decision function', 'true inliers', 'true outliers'], - prop=matplotlib.font_manager.FontProperties(size=11), + prop=matplotlib.font_manager.FontProperties(size=10), loc='lower right') - subplot.set_title("%d. %s (errors: %d)" % (i + 1, clf_name, n_errors)) + subplot.set_xlabel("%d. %s (errors: %d)" % (i + 1, clf_name, n_errors)) subplot.set_xlim((-7, 7)) subplot.set_ylim((-7, 7)) - plt.subplots_adjust(0.04, 0.1, 0.96, 0.92, 0.1, 0.26) + plt.subplots_adjust(0.04, 0.1, 0.96, 0.94, 0.1, 0.26) + plt.suptitle("Outlier detection") plt.show() diff --git a/examples/linear_model/plot_logistic.py b/examples/linear_model/plot_logistic.py index cdb95a7cfa563..488f1c3543a6a 100644 --- a/examples/linear_model/plot_logistic.py +++ b/examples/linear_model/plot_logistic.py @@ -4,12 +4,12 @@ """ ========================================================= -Logit function +Logistic function ========================================================= -Show in the plot is how the logistic regression would, in this +Shown in the plot is how the logistic regression would, in this synthetic dataset, classify values as either 0 or 1, -i.e. class one or two, using the logit-curve. +i.e. class one or two, using the logistic curve. """ print(__doc__) @@ -48,7 +48,7 @@ def model(x): return 1 / (1 + np.exp(-x)) loss = model(X_test * clf.coef_ + clf.intercept_).ravel() -plt.plot(X_test, loss, color='blue', linewidth=3) +plt.plot(X_test, loss, color='red', linewidth=3) ols = linear_model.LinearRegression() ols.fit(X, y) @@ -57,9 +57,10 @@ def model(x): plt.ylabel('y') plt.xlabel('X') -plt.xticks(()) -plt.yticks(()) +plt.xticks(range(-5, 10)) +plt.yticks([0, 0.5, 1]) plt.ylim(-.25, 1.25) plt.xlim(-4, 10) - +plt.legend(('Logistic Regression Model', 'Linear Regression Model'), + loc="lower right", fontsize='small') plt.show() diff --git a/examples/neighbors/plot_lof.py b/examples/neighbors/plot_lof.py new file mode 100644 index 0000000000000..cb5d349108467 --- /dev/null +++ b/examples/neighbors/plot_lof.py @@ -0,0 +1,56 @@ +""" +================================================= +Anomaly detection with Local Outlier Factor (LOF) +================================================= + +This example presents the Local Outlier Factor (LOF) estimator. The LOF +algorithm is an unsupervised outlier detection method which computes the local +density deviation of a given data point with respect to its neighbors. +It considers as outlier samples that have a substantially lower density than +their neighbors. + +The number of neighbors considered, (parameter n_neighbors) is typically +chosen 1) greater than the minimum number of objects a cluster has to contain, +so that other objects can be local outliers relative to this cluster, and 2) +smaller than the maximum number of close by objects that can potentially be +local outliers. +In practice, such informations are generally not available, and taking +n_neighbors=20 appears to work well in general. +""" + +import numpy as np +import matplotlib.pyplot as plt +from sklearn.neighbors import LocalOutlierFactor +print(__doc__) + +np.random.seed(42) + +# Generate train data +X = 0.3 * np.random.randn(100, 2) +# Generate some abnormal novel observations +X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2)) +X = np.r_[X + 2, X - 2, X_outliers] + +# fit the model +clf = LocalOutlierFactor(n_neighbors=20) +y_pred = clf.fit_predict(X) +y_pred_outliers = y_pred[200:] + +# plot the level sets of the decision function +xx, yy = np.meshgrid(np.linspace(-5, 5, 50), np.linspace(-5, 5, 50)) +Z = clf._decision_function(np.c_[xx.ravel(), yy.ravel()]) +Z = Z.reshape(xx.shape) + +plt.title("Local Outlier Factor (LOF)") +plt.contourf(xx, yy, Z, cmap=plt.cm.Blues_r) + +a = plt.scatter(X[:200, 0], X[:200, 1], c='white') +b = plt.scatter(X[200:, 0], X[200:, 1], c='red') +plt.axis('tight') +plt.xlim((-5, 5)) +plt.ylim((-5, 5)) +plt.legend([a, b], + ["normal observations", + "abnormal observations"], + loc="upper left") +plt.show() diff --git a/examples/text/document_classification_20newsgroups.py b/examples/text/document_classification_20newsgroups.py index 16fa8d7b66d94..a94a11b20a9b0 100644 --- a/examples/text/document_classification_20newsgroups.py +++ b/examples/text/document_classification_20newsgroups.py @@ -122,7 +122,8 @@ remove=remove) print('data loaded') -categories = data_train.target_names # for case categories == None +# order of labels in `target_names` can be different from `categories` +target_names = data_train.target_names def size_mb(docs): @@ -218,16 +219,15 @@ def benchmark(clf): if opts.print_top10 and feature_names is not None: print("top 10 keywords per class:") - for i, category in enumerate(categories): + for i, label in enumerate(target_names): top10 = np.argsort(clf.coef_[i])[-10:] - print(trim("%s: %s" - % (category, " ".join(feature_names[top10])))) + print(trim("%s: %s" % (label, " ".join(feature_names[top10])))) print() if opts.print_report: print("classification report:") print(metrics.classification_report(y_test, pred, - target_names=categories)) + target_names=target_names)) if opts.print_cm: print("confusion matrix:") diff --git a/sklearn/cluster/k_means_.py b/sklearn/cluster/k_means_.py index 7abd105926755..2596a47307711 100644 --- a/sklearn/cluster/k_means_.py +++ b/sklearn/cluster/k_means_.py @@ -18,7 +18,7 @@ from ..base import BaseEstimator, ClusterMixin, TransformerMixin from ..metrics.pairwise import euclidean_distances -from ..utils.extmath import row_norms, squared_norm +from ..utils.extmath import row_norms, squared_norm, stable_cumsum from ..utils.sparsefuncs_fast import assign_rows_csr from ..utils.sparsefuncs import mean_variance_axis from ..utils.fixes import astype @@ -106,7 +106,8 @@ def _k_init(X, n_clusters, x_squared_norms, random_state, n_local_trials=None): # Choose center candidates by sampling with probability proportional # to the squared distance to the closest existing center rand_vals = random_state.random_sample(n_local_trials) * current_pot - candidate_ids = np.searchsorted(closest_dist_sq.cumsum(), rand_vals) + candidate_ids = np.searchsorted(stable_cumsum(closest_dist_sq), + rand_vals) # Compute distances to center candidates distance_to_candidates = euclidean_distances( diff --git a/sklearn/covariance/outlier_detection.py b/sklearn/covariance/outlier_detection.py index 90e318073a364..1cafe885fdd47 100644 --- a/sklearn/covariance/outlier_detection.py +++ b/sklearn/covariance/outlier_detection.py @@ -106,26 +106,6 @@ class EllipticEnvelope(ClassifierMixin, OutlierDetectionMixin, MinCovDet): Read more in the :ref:`User Guide `. - Attributes - ---------- - `contamination` : float, 0. < contamination < 0.5 - The amount of contamination of the data set, i.e. the proportion of \ - outliers in the data set. - - location_ : array-like, shape (n_features,) - Estimated robust location - - covariance_ : array-like, shape (n_features, n_features) - Estimated robust covariance matrix - - precision_ : array-like, shape (n_features, n_features) - Estimated pseudo inverse matrix. - (stored only if store_precision is True) - - support_ : array-like, shape (n_samples,) - A mask of the observations that have been used to compute the - robust estimates of location and shape. - Parameters ---------- store_precision : bool @@ -150,6 +130,22 @@ class EllipticEnvelope(ClassifierMixin, OutlierDetectionMixin, MinCovDet): The amount of contamination of the data set, i.e. the proportion of outliers in the data set. + Attributes + ---------- + location_ : array-like, shape (n_features,) + Estimated robust location + + covariance_ : array-like, shape (n_features, n_features) + Estimated robust covariance matrix + + precision_ : array-like, shape (n_features, n_features) + Estimated pseudo inverse matrix. + (stored only if store_precision is True) + + support_ : array-like, shape (n_samples,) + A mask of the observations that have been used to compute the + robust estimates of location and shape. + See Also -------- EmpiricalCovariance, MinCovDet diff --git a/sklearn/cross_validation.py b/sklearn/cross_validation.py index 010f7106a4870..65960aaa9efe0 100644 --- a/sklearn/cross_validation.py +++ b/sklearn/cross_validation.py @@ -109,6 +109,10 @@ def _empty_mask(self): class LeaveOneOut(_PartitionIterator): """Leave-One-Out cross validation iterator. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.LeaveOneOut` instead. + Provides train/test indices to split data in train test sets. Each sample is used once as a test set (singleton) while the remaining samples form the training set. @@ -171,6 +175,10 @@ def __len__(self): class LeavePOut(_PartitionIterator): """Leave-P-Out cross validation iterator + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.LeavePOut` instead. + Provides train/test indices to split data in train test sets. This results in testing on all distinct samples of size p, while the remaining n - p samples form the training set in each iteration. @@ -266,6 +274,10 @@ def __init__(self, n, n_folds, shuffle, random_state): class KFold(_BaseKFold): """K-Folds cross validation iterator. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.KFold` instead. + Provides train/test indices to split data in train test sets. Split dataset into k consecutive folds (without shuffling by default). @@ -357,6 +369,10 @@ def __len__(self): class LabelKFold(_BaseKFold): """K-fold iterator variant with non-overlapping labels. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.GroupKFold` instead. + The same label will not appear in two different folds (the number of distinct labels has to be at least equal to the number of folds). @@ -459,6 +475,10 @@ def __len__(self): class StratifiedKFold(_BaseKFold): """Stratified K-Folds cross validation iterator + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.StratifiedKFold` instead. + Provides train/test indices to split data in train test sets. This cross-validation object is a variation of KFold that @@ -581,6 +601,10 @@ def __len__(self): class LeaveOneLabelOut(_PartitionIterator): """Leave-One-Label_Out cross-validation iterator + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.LeaveOneGroupOut` instead. + Provides train/test indices to split data according to a third-party provided label. This label information can be used to encode arbitrary domain specific stratifications of the samples as integers. @@ -651,6 +675,10 @@ def __len__(self): class LeavePLabelOut(_PartitionIterator): """Leave-P-Label_Out cross-validation iterator + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.LeavePGroupsOut` instead. + Provides train/test indices to split data according to a third-party provided label. This label information can be used to encode arbitrary domain specific stratifications of the samples as integers. @@ -762,6 +790,10 @@ def _iter_indices(self): class ShuffleSplit(BaseShuffleSplit): """Random permutation cross-validation iterator. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.ShuffleSplit` instead. + Yields indices to split data into training and test sets. Note: contrary to other cross-validation strategies, random splits @@ -963,6 +995,10 @@ def _approximate_mode(class_counts, n_draws, rng): class StratifiedShuffleSplit(BaseShuffleSplit): """Stratified ShuffleSplit cross validation iterator + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.StratifiedShuffleSplit` instead. + Provides train/test indices to split data in train test sets. This cross-validation object is a merge of StratifiedKFold and @@ -1085,6 +1121,10 @@ def __len__(self): class PredefinedSplit(_PartitionIterator): """Predefined split cross validation iterator + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.PredefinedSplit` instead. + Splits the data into training/test set folds according to a predefined scheme. Each sample can be assigned to at most one test set fold, as specified by the user through the ``test_fold`` parameter. @@ -1140,6 +1180,10 @@ def __len__(self): class LabelShuffleSplit(ShuffleSplit): """Shuffle-Labels-Out cross-validation iterator + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.GroupShuffleSplit` instead. + Provides randomized train/test indices to split data according to a third-party provided label. This label information can be used to encode arbitrary domain specific stratifications of the samples as integers. @@ -1241,6 +1285,10 @@ def cross_val_predict(estimator, X, y=None, cv=None, n_jobs=1, verbose=0, fit_params=None, pre_dispatch='2*n_jobs'): """Generate cross-validated estimates for each input data point + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :func:`sklearn.model_selection.cross_val_predict` instead. + Read more in the :ref:`User Guide `. Parameters @@ -1421,6 +1469,10 @@ def cross_val_score(estimator, X, y=None, scoring=None, cv=None, n_jobs=1, verbose=0, fit_params=None, pre_dispatch='2*n_jobs'): """Evaluate a score by cross-validation + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :func:`sklearn.model_selection.cross_val_score` instead. + Read more in the :ref:`User Guide `. Parameters @@ -1724,6 +1776,10 @@ def _shuffle(y, labels, random_state): def check_cv(cv, X=None, y=None, classifier=False): """Input checker utility for building a CV in a user friendly way. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :func:`sklearn.model_selection.check_cv` instead. + Parameters ---------- cv : int, cross-validation generator or an iterable, optional @@ -1781,6 +1837,10 @@ def permutation_test_score(estimator, X, y, cv=None, random_state=0, verbose=0, scoring=None): """Evaluate the significance of a cross-validated score with permutations + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :func:`sklearn.model_selection.permutation_test_score` instead. + Read more in the :ref:`User Guide `. Parameters @@ -1882,6 +1942,10 @@ def permutation_test_score(estimator, X, y, cv=None, def train_test_split(*arrays, **options): """Split arrays or matrices into random train and test subsets + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :func:`sklearn.model_selection.train_test_split` instead. + Quick utility that wraps input validation and ``next(iter(ShuffleSplit(n_samples)))`` and application to input data into a single call for splitting (and optionally subsampling) diff --git a/sklearn/datasets/lfw.py b/sklearn/datasets/lfw.py index af6503f14d7d1..e944e65867521 100644 --- a/sklearn/datasets/lfw.py +++ b/sklearn/datasets/lfw.py @@ -380,7 +380,13 @@ def _fetch_lfw_pairs(index_file_path, data_folder_path, slice_=None, "be removed in 0.19." "Use fetch_lfw_people(download_if_missing=False) instead.") def load_lfw_people(download_if_missing=False, **kwargs): - """Alias for fetch_lfw_people(download_if_missing=False) + """ + Alias for fetch_lfw_people(download_if_missing=False) + + .. deprecated:: 0.17 + This function will be removed in 0.19. + Use :func:`sklearn.datasets.fetch_lfw_people` with parameter + ``download_if_missing=False`` instead. Check fetch_lfw_people.__doc__ for the documentation and parameter list. """ @@ -509,7 +515,13 @@ def fetch_lfw_pairs(subset='train', data_home=None, funneled=True, resize=0.5, "be removed in 0.19." "Use fetch_lfw_pairs(download_if_missing=False) instead.") def load_lfw_pairs(download_if_missing=False, **kwargs): - """Alias for fetch_lfw_pairs(download_if_missing=False) + """ + Alias for fetch_lfw_pairs(download_if_missing=False) + + .. deprecated:: 0.17 + This function will be removed in 0.19. + Use :func:`sklearn.datasets.fetch_lfw_pairs` with parameter + ``download_if_missing=False`` instead. Check fetch_lfw_pairs.__doc__ for the documentation and parameter list. """ diff --git a/sklearn/datasets/samples_generator.py b/sklearn/datasets/samples_generator.py index 53ee8987ba968..acd07337541ca 100644 --- a/sklearn/datasets/samples_generator.py +++ b/sklearn/datasets/samples_generator.py @@ -1194,7 +1194,7 @@ def make_sparse_spd_matrix(dim=1, alpha=0.95, norm_diag=False, The size of the random matrix to generate. alpha : float between 0 and 1, optional (default=0.95) - The probability that a coefficient is zero (see notes). Larger values + The probability that a coefficient is zero (see notes). Larger values enforce more sparsity. random_state : int, RandomState instance or None, optional (default=None) diff --git a/sklearn/decomposition/pca.py b/sklearn/decomposition/pca.py index b9b171301faeb..9edfbded483a8 100644 --- a/sklearn/decomposition/pca.py +++ b/sklearn/decomposition/pca.py @@ -15,6 +15,7 @@ import numpy as np from scipy import linalg from scipy.special import gammaln +from scipy.sparse import issparse from ..externals import six @@ -24,6 +25,7 @@ from ..utils import check_random_state, as_float_array from ..utils import check_array from ..utils.extmath import fast_dot, fast_logdet, randomized_svd, svd_flip +from ..utils.extmath import stable_cumsum from ..utils.validation import check_is_fitted from ..utils.arpack import svds @@ -115,6 +117,9 @@ class PCA(_BasePCA): It can also use the scipy.sparse.linalg ARPACK implementation of the truncated SVD. + Notice that this class does not support sparse input. See + :class:`TruncatedSVD` for an alternative with sparse data. + Read more in the :ref:`User Guide `. Parameters @@ -331,6 +336,13 @@ def fit_transform(self, X, y=None): def _fit(self, X): """Dispatch to the right submethod depending on the chosen solver.""" + + # Raise an error for sparse input. + # This is more informative than the generic one raised by check_array. + if issparse(X): + raise TypeError('PCA does not support sparse input. See ' + 'TruncatedSVD for a possible alternative.') + X = check_array(X, dtype=[np.float64], ensure_2d=True, copy=self.copy) @@ -393,7 +405,7 @@ def _fit_full(self, X, n_components): elif 0 < n_components < 1.0: # number of components for which the cumulated explained # variance percentage is superior to the desired threshold - ratio_cumsum = explained_variance_ratio_.cumsum() + ratio_cumsum = stable_cumsum(explained_variance_ratio_) n_components = np.searchsorted(ratio_cumsum, n_components) + 1 # Compute noise covariance using Probabilistic PCA model @@ -526,6 +538,12 @@ def score(self, X, y=None): class RandomizedPCA(BaseEstimator, TransformerMixin): """Principal component analysis (PCA) using randomized SVD + .. deprecated:: 0.18 + This class will be removed in 0.20. + Use :class:`PCA` with parameter svd_solver 'randomized' instead. + The new implementation DOES NOT store whiten ``components_``. + Apply transform to get them. + Linear dimensionality reduction using approximated Singular Value Decomposition of the data and keeping only the most significant singular vectors to project the data to a lower dimensional space. diff --git a/sklearn/decomposition/tests/test_nmf.py b/sklearn/decomposition/tests/test_nmf.py index 43ca1423b92c3..c431dd3842be3 100644 --- a/sklearn/decomposition/tests/test_nmf.py +++ b/sklearn/decomposition/tests/test_nmf.py @@ -68,7 +68,7 @@ def test_initialize_variants(): random_state=0) for ref, evl in ((W0, Wa), (W0, War), (H0, Ha), (H0, Har)): - assert_true(np.allclose(evl[ref != 0], ref[ref != 0])) + assert_almost_equal(evl[ref != 0], ref[ref != 0]) @ignore_warnings @@ -128,9 +128,10 @@ def test_nmf_transform_custom_init(): H_init = np.abs(avg * random_state.randn(n_components, 5)) W_init = np.abs(avg * random_state.randn(6, n_components)) - m = NMF(solver='cd', n_components=n_components, init='custom', random_state=0) - ft = m.fit_transform(A, W=W_init, H=H_init) - t = m.transform(A) + m = NMF(solver='cd', n_components=n_components, init='custom', + random_state=0) + m.fit_transform(A, W=W_init, H=H_init) + m.transform(A) @ignore_warnings @@ -140,7 +141,7 @@ def test_nmf_inverse_transform(): A = np.abs(random_state.randn(6, 4)) for solver in ('pg', 'cd'): m = NMF(solver=solver, n_components=4, init='random', random_state=0) - ft = m.fit_transform(A) + m.fit_transform(A) t = m.transform(A) A_new = m.inverse_transform(t) assert_array_almost_equal(A, A_new, decimal=2) @@ -235,9 +236,11 @@ def test_non_negative_factorization_checking(): # Test parameters checking is public function nnmf = non_negative_factorization assert_no_warnings(nnmf, A, A, A, np.int64(1)) - msg = "Number of components must be a positive integer; got (n_components=1.5)" + msg = ("Number of components must be a positive integer; " + "got (n_components=1.5)") assert_raise_message(ValueError, msg, nnmf, A, A, A, 1.5) - msg = "Number of components must be a positive integer; got (n_components='2')" + msg = ("Number of components must be a positive integer; " + "got (n_components='2')") assert_raise_message(ValueError, msg, nnmf, A, A, A, '2') msg = "Negative values in data passed to NMF (input H)" assert_raise_message(ValueError, msg, nnmf, A, A, -A, 2, 'custom') diff --git a/sklearn/decomposition/tests/test_pca.py b/sklearn/decomposition/tests/test_pca.py index 58113f10f0674..befb9e8a55738 100644 --- a/sklearn/decomposition/tests/test_pca.py +++ b/sklearn/decomposition/tests/test_pca.py @@ -1,4 +1,5 @@ import numpy as np +import scipy as sp from itertools import product from sklearn.utils.testing import assert_almost_equal @@ -508,3 +509,15 @@ def fit_deprecated(X): assert_warns_message(DeprecationWarning, depr_message, fit_deprecated, X) Y_pca = PCA(svd_solver='randomized', random_state=0).fit_transform(X) assert_array_almost_equal(Y, Y_pca) + + +def test_pca_spase_input(): + + X = np.random.RandomState(0).rand(5, 4) + X = sp.sparse.csr_matrix(X) + assert(sp.sparse.issparse(X)) + + for svd_solver in solver_list: + pca = PCA(n_components=3, svd_solver=svd_solver) + + assert_raises(TypeError, pca.fit, X) diff --git a/sklearn/decomposition/truncated_svd.py b/sklearn/decomposition/truncated_svd.py index 4cd494ec5d7bd..1c86550f7571e 100644 --- a/sklearn/decomposition/truncated_svd.py +++ b/sklearn/decomposition/truncated_svd.py @@ -26,15 +26,16 @@ class TruncatedSVD(BaseEstimator, TransformerMixin): """Dimensionality reduction using truncated SVD (aka LSA). This transformer performs linear dimensionality reduction by means of - truncated singular value decomposition (SVD). It is very similar to PCA, - but operates on sample vectors directly, instead of on a covariance matrix. - This means it can work with scipy.sparse matrices efficiently. + truncated singular value decomposition (SVD). Contrary to PCA, this + estimator does not center the data before computing the singular value + decomposition. This means it can work with scipy.sparse matrices + efficiently. In particular, truncated SVD works on term count/tf-idf matrices as returned by the vectorizers in sklearn.feature_extraction.text. In that context, it is known as latent semantic analysis (LSA). - This estimator supports two algorithm: a fast randomized SVD solver, and + This estimator supports two algorithms: a fast randomized SVD solver, and a "naive" algorithm that uses ARPACK as an eigensolver on (X * X.T) or (X.T * X), whichever is more efficient. diff --git a/sklearn/discriminant_analysis.py b/sklearn/discriminant_analysis.py index 5c8f1c5d039a2..628314a013494 100644 --- a/sklearn/discriminant_analysis.py +++ b/sklearn/discriminant_analysis.py @@ -143,9 +143,6 @@ class LinearDiscriminantAnalysis(BaseEstimator, LinearClassifierMixin, .. versionadded:: 0.17 *LinearDiscriminantAnalysis*. - .. versionchanged:: 0.17 - Deprecated :class:`lda.LDA` have been moved to :class:`LinearDiscriminantAnalysis`. - Read more in the :ref:`User Guide `. Parameters @@ -293,8 +290,8 @@ def _solve_lsqr(self, X, y, shrinkage): self.means_ = _class_means(X, y) self.covariance_ = _class_cov(X, y, self.priors_, shrinkage) self.coef_ = linalg.lstsq(self.covariance_, self.means_.T)[0].T - self.intercept_ = (-0.5 * np.diag(np.dot(self.means_, self.coef_.T)) - + np.log(self.priors_)) + self.intercept_ = (-0.5 * np.diag(np.dot(self.means_, self.coef_.T)) + + np.log(self.priors_)) def _solve_eigen(self, X, y, shrinkage): """Eigenvalue solver. @@ -336,15 +333,16 @@ class scatter). This solver supports both classification and Sb = St - Sw # between scatter evals, evecs = linalg.eigh(Sb, Sw) - self.explained_variance_ratio_ = np.sort(evals / np.sum(evals))[::-1] + self.explained_variance_ratio_ = np.sort(evals / np.sum(evals) + )[::-1][:self._max_components] evecs = evecs[:, np.argsort(evals)[::-1]] # sort eigenvectors # evecs /= np.linalg.norm(evecs, axis=0) # doesn't work with numpy 1.6 evecs /= np.apply_along_axis(np.linalg.norm, 0, evecs) self.scalings_ = evecs self.coef_ = np.dot(self.means_, evecs).dot(evecs.T) - self.intercept_ = (-0.5 * np.diag(np.dot(self.means_, self.coef_.T)) - + np.log(self.priors_)) + self.intercept_ = (-0.5 * np.diag(np.dot(self.means_, self.coef_.T)) + + np.log(self.priors_)) def _solve_svd(self, X, y): """SVD solver. @@ -400,12 +398,12 @@ def _solve_svd(self, X, y): _, S, V = linalg.svd(X, full_matrices=0) self.explained_variance_ratio_ = (S**2 / np.sum( - S**2))[:self.n_components] + S**2))[:self._max_components] rank = np.sum(S > self.tol * S[0]) self.scalings_ = np.dot(scalings, V.T[:, :rank]) coef = np.dot(self.means_ - self.xbar_, self.scalings_) - self.intercept_ = (-0.5 * np.sum(coef ** 2, axis=1) - + np.log(self.priors_)) + self.intercept_ = (-0.5 * np.sum(coef ** 2, axis=1) + + np.log(self.priors_)) self.coef_ = np.dot(coef, self.scalings_.T) self.intercept_ -= np.dot(self.xbar_, self.coef_.T) @@ -457,6 +455,13 @@ def fit(self, X, y, store_covariance=None, tol=None): UserWarning) self.priors_ = self.priors_ / self.priors_.sum() + # Get the maximum number of components + if self.n_components is None: + self._max_components = len(self.classes_) - 1 + else: + self._max_components = min(len(self.classes_) - 1, + self.n_components) + if self.solver == 'svd': if self.shrinkage is not None: raise NotImplementedError('shrinkage not supported') @@ -497,9 +502,8 @@ def transform(self, X): X_new = np.dot(X - self.xbar_, self.scalings_) elif self.solver == 'eigen': X_new = np.dot(X, self.scalings_) - n_components = X.shape[1] if self.n_components is None \ - else self.n_components - return X_new[:, :n_components] + + return X_new[:, :self._max_components] def predict_proba(self, X): """Estimate probability. @@ -555,9 +559,6 @@ class QuadraticDiscriminantAnalysis(BaseEstimator, ClassifierMixin): .. versionadded:: 0.17 *QuadraticDiscriminantAnalysis* - .. versionchanged:: 0.17 - Deprecated :class:`qda.QDA` have been moved to :class:`QuadraticDiscriminantAnalysis`. - Read more in the :ref:`User Guide `. Parameters diff --git a/sklearn/ensemble/_gradient_boosting.pyx b/sklearn/ensemble/_gradient_boosting.pyx index 9e6e9f6d29c0e..71371f5c24a48 100644 --- a/sklearn/ensemble/_gradient_boosting.pyx +++ b/sklearn/ensemble/_gradient_boosting.pyx @@ -8,13 +8,22 @@ cimport cython +from libc.stdlib cimport free +from libc.string cimport memset + import numpy as np cimport numpy as np np.import_array() +from scipy.sparse import issparse +from scipy.sparse import csr_matrix + from sklearn.tree._tree cimport Node from sklearn.tree._tree cimport Tree - +from sklearn.tree._tree cimport DTYPE_t +from sklearn.tree._tree cimport SIZE_t +from sklearn.tree._tree cimport INT32_t +from sklearn.tree._utils cimport safe_realloc ctypedef np.int32_t int32 ctypedef np.float64_t float64 @@ -27,24 +36,19 @@ from numpy import bool as np_bool from numpy import float32 as np_float32 from numpy import float64 as np_float64 -# Define a datatype for the data array -DTYPE = np.float32 -ctypedef np.float32_t DTYPE_t -ctypedef np.npy_intp SIZE_t - # constant to mark tree leafs -cdef int LEAF = -1 - -cdef void _predict_regression_tree_inplace_fast(DTYPE_t *X, - Node* root_node, - double *value, - double scale, - Py_ssize_t k, - Py_ssize_t K, - Py_ssize_t n_samples, - Py_ssize_t n_features, - float64 *out): +cdef SIZE_t TREE_LEAF = -1 + +cdef void _predict_regression_tree_inplace_fast_dense(DTYPE_t *X, + Node* root_node, + double *value, + double scale, + Py_ssize_t k, + Py_ssize_t K, + Py_ssize_t n_samples, + Py_ssize_t n_features, + float64 *out): """Predicts output for regression tree and stores it in ``out[i, k]``. This function operates directly on the data arrays of the tree @@ -84,22 +88,108 @@ cdef void _predict_regression_tree_inplace_fast(DTYPE_t *X, shape ``(n_samples, K)``. """ cdef Py_ssize_t i - cdef int32 node_id cdef Node *node for i in range(n_samples): node = root_node # While node not a leaf - while node.left_child != -1 and node.right_child != -1: + while node.left_child != TREE_LEAF: if X[i * n_features + node.feature] <= node.threshold: node = root_node + node.left_child else: node = root_node + node.right_child out[i * K + k] += scale * value[node - root_node] +def _predict_regression_tree_stages_sparse(np.ndarray[object, ndim=2] estimators, + object X, double scale, + np.ndarray[float64, ndim=2] out): + """Predicts output for regression tree inplace and adds scaled value to ``out[i, k]``. + + The function assumes that the ndarray that wraps ``X`` is csr_matrix. + """ + cdef DTYPE_t* X_data = ( X.data).data + cdef INT32_t* X_indices = ( X.indices).data + cdef INT32_t* X_indptr = ( X.indptr).data + + cdef SIZE_t n_samples = X.shape[0] + cdef SIZE_t n_features = X.shape[1] + cdef SIZE_t n_stages = estimators.shape[0] + cdef SIZE_t n_outputs = estimators.shape[1] + + # Initialize output + cdef float64* out_ptr = out.data + + # Indices and temporary variables + cdef SIZE_t sample_i + cdef SIZE_t feature_i + cdef SIZE_t stage_i + cdef SIZE_t output_i + cdef Node *root_node = NULL + cdef Node *node = NULL + cdef double *value = NULL + + cdef Tree tree + cdef Node** nodes = NULL + cdef double** values = NULL + safe_realloc(&nodes, n_stages * n_outputs) + safe_realloc(&values, n_stages * n_outputs) + for stage_i in range(n_stages): + for output_i in range(n_outputs): + tree = estimators[stage_i, output_i].tree_ + nodes[stage_i * n_outputs + output_i] = tree.nodes + values[stage_i * n_outputs + output_i] = tree.value + + # Initialize auxiliary data-structure + cdef DTYPE_t feature_value = 0. + cdef DTYPE_t* X_sample = NULL + + # feature_to_sample as a data structure records the last seen sample + # for each feature; functionally, it is an efficient way to identify + # which features are nonzero in the present sample. + cdef SIZE_t* feature_to_sample = NULL + + safe_realloc(&X_sample, n_features) + safe_realloc(&feature_to_sample, n_features) + + memset(feature_to_sample, -1, n_features * sizeof(SIZE_t)) + + # Cycle through all samples + for sample_i in range(n_samples): + for feature_i in range(X_indptr[sample_i], X_indptr[sample_i + 1]): + feature_to_sample[X_indices[feature_i]] = sample_i + X_sample[X_indices[feature_i]] = X_data[feature_i] + + # Cycle through all stages + for stage_i in range(n_stages): + # Cycle through all trees + for output_i in range(n_outputs): + root_node = nodes[stage_i * n_outputs + output_i] + value = values[stage_i * n_outputs + output_i] + node = root_node + + # While node not a leaf + while node.left_child != TREE_LEAF: + # ... and node.right_child != TREE_LEAF: + if feature_to_sample[node.feature] == sample_i: + feature_value = X_sample[node.feature] + else: + feature_value = 0. + + if feature_value <= node.threshold: + node = root_node + node.left_child + else: + node = root_node + node.right_child + out_ptr[sample_i * n_outputs + output_i] += (scale + * value[node - root_node]) + + # Free auxiliary arrays + free(X_sample) + free(feature_to_sample) + free(nodes) + free(values) + -@cython.nonecheck(False) def predict_stages(np.ndarray[object, ndim=2] estimators, - np.ndarray[DTYPE_t, ndim=2, mode='c'] X, double scale, + object X, double scale, np.ndarray[float64, ndim=2] out): """Add predictions of ``estimators`` to ``out``. @@ -112,25 +202,31 @@ def predict_stages(np.ndarray[object, ndim=2] estimators, cdef Py_ssize_t K = estimators.shape[1] cdef Tree tree - for i in range(n_estimators): - for k in range(K): - tree = estimators[i, k].tree_ + if issparse(X): + _predict_regression_tree_stages_sparse(estimators, X, scale, out) + else: + if not isinstance(X, np.ndarray): + raise ValueError("X should be in np.ndarray or csr_matrix format," + "got %s" % type(X)) + + for i in range(n_estimators): + for k in range(K): + tree = estimators[i, k].tree_ - # avoid buffer validation by casting to ndarray - # and get data pointer - # need brackets because of casting operator priority - _predict_regression_tree_inplace_fast( - X.data, - tree.nodes, tree.value, - scale, k, K, X.shape[0], X.shape[1], - ( out).data) - ## out += scale * tree.predict(X).reshape((X.shape[0], 1)) + # avoid buffer validation by casting to ndarray + # and get data pointer + # need brackets because of casting operator priority + _predict_regression_tree_inplace_fast_dense( + ( X).data, + tree.nodes, tree.value, + scale, k, K, X.shape[0], X.shape[1], + ( out).data) + ## out += scale * tree.predict(X).reshape((X.shape[0], 1)) -@cython.nonecheck(False) def predict_stage(np.ndarray[object, ndim=2] estimators, int stage, - np.ndarray[DTYPE_t, ndim=2] X, double scale, + object X, double scale, np.ndarray[float64, ndim=2] out): """Add predictions of ``estimators[stage]`` to ``out``. @@ -216,7 +312,7 @@ cpdef _partial_dependence_tree(Tree tree, DTYPE_t[:, ::1] X, stack_size -= 1 current_node = node_stack[stack_size] - if current_node.left_child == LEAF: + if current_node.left_child == TREE_LEAF: out[i] += weight_stack[stack_size] * value[current_node - root_node] * \ learn_rate total_weight += weight_stack[stack_size] diff --git a/sklearn/ensemble/base.py b/sklearn/ensemble/base.py index 2add9d730062e..b965988a4879a 100644 --- a/sklearn/ensemble/base.py +++ b/sklearn/ensemble/base.py @@ -6,6 +6,7 @@ # License: BSD 3 clause import numpy as np +import numbers from ..base import clone from ..base import BaseEstimator @@ -93,6 +94,10 @@ def __init__(self, base_estimator, n_estimators=10, def _validate_estimator(self, default=None): """Check the estimator and the n_estimator attribute, set the `base_estimator_` attribute.""" + if not isinstance(self.n_estimators, (numbers.Integral, np.integer)): + raise ValueError("n_estimators must be an integer, " + "got {0}.".format(type(self.n_estimators))) + if self.n_estimators <= 0: raise ValueError("n_estimators must be greater than zero, " "got {0}.".format(self.n_estimators)) diff --git a/sklearn/ensemble/gradient_boosting.py b/sklearn/ensemble/gradient_boosting.py index c9a36aac1bd99..fcfeb45a09157 100644 --- a/sklearn/ensemble/gradient_boosting.py +++ b/sklearn/ensemble/gradient_boosting.py @@ -1158,8 +1158,10 @@ def _staged_decision_function(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Returns ------- @@ -1169,7 +1171,7 @@ def _staged_decision_function(self, X): Regression and binary classification are special cases with ``k == 1``, otherwise ``k==n_classes``. """ - X = check_array(X, dtype=DTYPE, order="C") + X = check_array(X, dtype=DTYPE, order="C", accept_sparse='csr') score = self._init_decision_function(X) for i in range(self.estimators_.shape[0]): predict_stage(self.estimators_, i, X, self.learning_rate, score) @@ -1479,8 +1481,10 @@ def decision_function(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Returns ------- @@ -1490,7 +1494,7 @@ def decision_function(self, X): Regression and binary classification produce an array of shape [n_samples]. """ - X = check_array(X, dtype=DTYPE, order="C") + X = check_array(X, dtype=DTYPE, order="C", accept_sparse='csr') score = self._decision_function(X) if score.shape[1] == 1: return score.ravel() @@ -1504,8 +1508,10 @@ def staged_decision_function(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Returns ------- @@ -1524,8 +1530,10 @@ def predict(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Returns ------- @@ -1544,8 +1552,10 @@ def staged_predict(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Returns ------- @@ -1561,8 +1571,10 @@ def predict_proba(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Raises ------ @@ -1589,8 +1601,10 @@ def predict_log_proba(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Raises ------ @@ -1614,8 +1628,10 @@ def staged_predict_proba(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Returns ------- @@ -1840,15 +1856,17 @@ def predict(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Returns ------- y : array of shape = [n_samples] The predicted values. """ - X = check_array(X, dtype=DTYPE, order="C") + X = check_array(X, dtype=DTYPE, order="C", accept_sparse='csr') return self._decision_function(X).ravel() def staged_predict(self, X): @@ -1859,8 +1877,10 @@ def staged_predict(self, X): Parameters ---------- - X : array-like of shape = [n_samples, n_features] - The input samples. + X : array-like or sparse matrix, shape = [n_samples, n_features] + The input samples. Internally, it will be converted to + ``dtype=np.float32`` and if a sparse matrix is provided + to a sparse ``csr_matrix``. Returns ------- diff --git a/sklearn/ensemble/partial_dependence.py b/sklearn/ensemble/partial_dependence.py index 0f0fcd9cab552..b7e7f6ccccfe7 100644 --- a/sklearn/ensemble/partial_dependence.py +++ b/sklearn/ensemble/partial_dependence.py @@ -183,10 +183,14 @@ def plot_partial_dependence(gbrt, X, features, feature_names=None, A fitted gradient boosting model. X : array-like, shape=(n_samples, n_features) The data on which ``gbrt`` was trained. - features : seq of tuples or ints + features : seq of ints, strings, or tuples of ints or strings If seq[i] is an int or a tuple with one int value, a one-way PDP is created; if seq[i] is a tuple of two ints, a two-way PDP is created. + If feature_names is specified and seq[i] is an int, seq[i] + must be < len(feature_names). + If seq[i] is a string, feature_names must be specified, and + seq[i] must be in feature_names. feature_names : seq of str Name of each feature; feature_names[i] holds the name of the feature with index i. @@ -306,8 +310,9 @@ def convert_feature(fx): l.append(feature_names[i]) names.append(l) except IndexError: - raise ValueError('features[i] must be in [0, n_features) ' - 'but was %d' % i) + raise ValueError('All entries of features must be less than ' + 'len(feature_names) = {0}, got {1}.' + .format(len(feature_names), i)) # compute PD functions pd_result = Parallel(n_jobs=n_jobs, verbose=verbose)( diff --git a/sklearn/ensemble/tests/test_base.py b/sklearn/ensemble/tests/test_base.py index 8484283fcc0a0..bed9a3a5e8122 100644 --- a/sklearn/ensemble/tests/test_base.py +++ b/sklearn/ensemble/tests/test_base.py @@ -46,17 +46,38 @@ def test_base(): assert_true(isinstance(ensemble[2].random_state, int)) assert_not_equal(ensemble[1].random_state, ensemble[2].random_state) + np_int_ensemble = BaggingClassifier(base_estimator=Perceptron(), + n_estimators=np.int32(3)) + np_int_ensemble.fit(iris.data, iris.target) + def test_base_zero_n_estimators(): # Check that instantiating a BaseEnsemble with n_estimators<=0 raises # a ValueError. - ensemble = BaggingClassifier(base_estimator=Perceptron(), n_estimators=0) + ensemble = BaggingClassifier(base_estimator=Perceptron(), + n_estimators=0) iris = load_iris() assert_raise_message(ValueError, "n_estimators must be greater than zero, got 0.", ensemble.fit, iris.data, iris.target) +def test_base_not_int_n_estimators(): + # Check that instantiating a BaseEnsemble with a string as n_estimators + # raises a ValueError demanding n_estimators to be supplied as an integer. + string_ensemble = BaggingClassifier(base_estimator=Perceptron(), + n_estimators='3') + iris = load_iris() + assert_raise_message(ValueError, + "n_estimators must be an integer", + string_ensemble.fit, iris.data, iris.target) + float_ensemble = BaggingClassifier(base_estimator=Perceptron(), + n_estimators=3.0) + assert_raise_message(ValueError, + "n_estimators must be an integer", + float_ensemble.fit, iris.data, iris.target) + + def test_set_random_states(): # Linear Discriminant Analysis doesn't have random state: smoke test _set_random_states(LinearDiscriminantAnalysis(), random_state=17) diff --git a/sklearn/ensemble/tests/test_gradient_boosting.py b/sklearn/ensemble/tests/test_gradient_boosting.py index 1ebf82fa3ae44..634bc259a1167 100644 --- a/sklearn/ensemble/tests/test_gradient_boosting.py +++ b/sklearn/ensemble/tests/test_gradient_boosting.py @@ -1050,6 +1050,9 @@ def check_sparse_input(EstimatorClass, X, X_sparse, y): assert_array_almost_equal(sparse.feature_importances_, auto.feature_importances_) + assert_array_almost_equal(sparse.predict(X_sparse), dense.predict(X)) + assert_array_almost_equal(dense.predict(X_sparse), sparse.predict(X)) + if isinstance(EstimatorClass, GradientBoostingClassifier): assert_array_almost_equal(sparse.predict_proba(X), dense.predict_proba(X)) @@ -1061,6 +1064,14 @@ def check_sparse_input(EstimatorClass, X, X_sparse, y): assert_array_almost_equal(sparse.predict_log_proba(X), auto.predict_log_proba(X)) + assert_array_almost_equal(sparse.decision_function(X_sparse), + sparse.decision_function(X)) + assert_array_almost_equal(dense.decision_function(X_sparse), + sparse.decision_function(X)) + + assert_array_almost_equal( + np.array(sparse.staged_decision_function(X_sparse)), + np.array(sparse.staged_decision_function(X))) @skip_if_32bit def test_sparse_input(): diff --git a/sklearn/ensemble/weight_boosting.py b/sklearn/ensemble/weight_boosting.py index 56d7d6ff80a2f..16afc4311e90b 100644 --- a/sklearn/ensemble/weight_boosting.py +++ b/sklearn/ensemble/weight_boosting.py @@ -38,6 +38,7 @@ from ..tree.tree import BaseDecisionTree from ..tree._tree import DTYPE from ..utils import check_array, check_X_y, check_random_state +from ..utils.extmath import stable_cumsum from ..metrics import accuracy_score, r2_score from sklearn.utils.validation import has_fit_parameter, check_is_fitted @@ -1002,7 +1003,7 @@ def _boost(self, iboost, X, y, sample_weight, random_state): # Weighted sampling of the training set with replacement # For NumPy >= 1.7.0 use np.random.choice - cdf = sample_weight.cumsum() + cdf = stable_cumsum(sample_weight) cdf /= cdf[-1] uniform_samples = random_state.random_sample(X.shape[0]) bootstrap_idx = cdf.searchsorted(uniform_samples, side='right') @@ -1059,7 +1060,7 @@ def _get_median_predict(self, X, limit): sorted_idx = np.argsort(predictions, axis=1) # Find index of median prediction for each sample - weight_cdf = self.estimator_weights_[sorted_idx].cumsum(axis=1) + weight_cdf = stable_cumsum(self.estimator_weights_[sorted_idx], axis=1) median_or_above = weight_cdf >= 0.5 * weight_cdf[:, -1][:, np.newaxis] median_idx = median_or_above.argmax(axis=1) diff --git a/sklearn/externals/joblib/__init__.py b/sklearn/externals/joblib/__init__.py index 43000626b2cf4..ce1957d1def5a 100644 --- a/sklearn/externals/joblib/__init__.py +++ b/sklearn/externals/joblib/__init__.py @@ -115,7 +115,8 @@ # Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer. # 'X.Y.dev0' is the canonical version of 'X.Y.dev' # -__version__ = '0.10.2' + +__version__ = '0.10.3' from .memory import Memory, MemorizedResult diff --git a/sklearn/externals/joblib/_parallel_backends.py b/sklearn/externals/joblib/_parallel_backends.py index e9325830479c7..cc4f221d21282 100644 --- a/sklearn/externals/joblib/_parallel_backends.py +++ b/sklearn/externals/joblib/_parallel_backends.py @@ -265,12 +265,16 @@ def effective_n_jobs(self, n_jobs): This also checks if we are attempting to create a nested parallel loop. """ + if mp is None: + return 1 + if mp.current_process().daemon: # Daemonic processes cannot have children - warnings.warn( - 'Multiprocessing-backed parallel loops cannot be nested,' - ' setting n_jobs=1', - stacklevel=3) + if n_jobs != 1: + warnings.warn( + 'Multiprocessing-backed parallel loops cannot be nested,' + ' setting n_jobs=1', + stacklevel=3) return 1 elif threading.current_thread().name != 'MainThread': diff --git a/sklearn/externals/joblib/testing.py b/sklearn/externals/joblib/testing.py index 21dfbc8689af4..94c023c3f0aed 100644 --- a/sklearn/externals/joblib/testing.py +++ b/sklearn/externals/joblib/testing.py @@ -51,10 +51,12 @@ def assert_raises_regex(expected_exception, expected_regexp, expected_exception(expected_regexp)) -def check_subprocess_call(cmd, timeout=1, stdout_regex=None): +def check_subprocess_call(cmd, timeout=1, stdout_regex=None, + stderr_regex=None): """Runs a command in a subprocess with timeout in seconds. - Also checks returncode is zero and stdout if stdout_regex is set. + Also checks returncode is zero, stdout if stdout_regex is set, and + stderr if stderr_regex is set. """ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -79,7 +81,13 @@ def kill_process(): if (stdout_regex is not None and not re.search(stdout_regex, stdout)): raise ValueError( - "Unexpected output: '{0!r}' does not match:\n{1!r}".format( + "Unexpected stdout: {0!r} does not match:\n{1!r}".format( stdout_regex, stdout)) + if (stderr_regex is not None and + not re.search(stderr_regex, stderr)): + raise ValueError( + "Unexpected stderr: {0!r} does not match:\n{1!r}".format( + stderr_regex, stderr)) + finally: timer.cancel() diff --git a/sklearn/feature_selection/from_model.py b/sklearn/feature_selection/from_model.py index c8a0679247f16..e8a18031f1dc8 100644 --- a/sklearn/feature_selection/from_model.py +++ b/sklearn/feature_selection/from_model.py @@ -10,9 +10,10 @@ from ..utils import safe_mask, check_array, deprecated from ..utils.validation import check_is_fitted from ..exceptions import NotFittedError +from ..utils.fixes import norm -def _get_feature_importances(estimator): +def _get_feature_importances(estimator, norm_order=1): """Retrieve or aggregate feature importances from estimator""" importances = getattr(estimator, "feature_importances_", None) @@ -21,7 +22,7 @@ def _get_feature_importances(estimator): importances = np.abs(estimator.coef_) else: - importances = np.sum(np.abs(estimator.coef_), axis=0) + importances = norm(estimator.coef_, axis=0, ord=norm_order) elif importances is None: raise ValueError( @@ -172,6 +173,11 @@ class SelectFromModel(BaseEstimator, SelectorMixin): Otherwise train the model using ``fit`` and then ``transform`` to do feature selection. + norm_order : non-zero int, inf, -inf, default 1 + Order of the norm used to filter the vectors of coefficients below + ``threshold`` in the case where the ``coef_`` attribute of the + estimator is of dimension 2. + Attributes ---------- `estimator_`: an estimator @@ -182,10 +188,12 @@ class SelectFromModel(BaseEstimator, SelectorMixin): `threshold_`: float The threshold value used for feature selection. """ - def __init__(self, estimator, threshold=None, prefit=False): + + def __init__(self, estimator, threshold=None, prefit=False, norm_order=1): self.estimator = estimator self.threshold = threshold self.prefit = prefit + self.norm_order = norm_order def _get_support_mask(self): # SelectFromModel can directly call on transform. @@ -197,7 +205,7 @@ def _get_support_mask(self): raise ValueError( 'Either fit the model before transform or set "prefit=True"' ' while passing the fitted estimator to the constructor.') - scores = _get_feature_importances(estimator) + scores = _get_feature_importances(estimator, self.norm_order) self.threshold_ = _calculate_threshold(estimator, scores, self.threshold) return scores >= self.threshold_ diff --git a/sklearn/feature_selection/tests/test_feature_select.py b/sklearn/feature_selection/tests/test_feature_select.py index d2a73334299d5..d08008d31c304 100644 --- a/sklearn/feature_selection/tests/test_feature_select.py +++ b/sklearn/feature_selection/tests/test_feature_select.py @@ -371,6 +371,40 @@ def test_select_heuristics_regression(): assert_less(np.sum(support[5:] == 1), 3) +def test_boundary_case_ch2(): + # Test boundary case, and always aim to select 1 feature. + X = np.array([[10, 20], [20, 20], [20, 30]]) + y = np.array([[1], [0], [0]]) + scores, pvalues = chi2(X, y) + assert_array_almost_equal(scores, np.array([4., 0.71428571])) + assert_array_almost_equal(pvalues, np.array([0.04550026, 0.39802472])) + + filter_fdr = SelectFdr(chi2, alpha=0.1) + filter_fdr.fit(X, y) + support_fdr = filter_fdr.get_support() + assert_array_equal(support_fdr, np.array([True, False])) + + filter_kbest = SelectKBest(chi2, k=1) + filter_kbest.fit(X, y) + support_kbest = filter_kbest.get_support() + assert_array_equal(support_kbest, np.array([True, False])) + + filter_percentile = SelectPercentile(chi2, percentile=50) + filter_percentile.fit(X, y) + support_percentile = filter_percentile.get_support() + assert_array_equal(support_percentile, np.array([True, False])) + + filter_fpr = SelectFpr(chi2, alpha=0.1) + filter_fpr.fit(X, y) + support_fpr = filter_fpr.get_support() + assert_array_equal(support_fpr, np.array([True, False])) + + filter_fwe = SelectFwe(chi2, alpha=0.1) + filter_fwe.fit(X, y) + support_fwe = filter_fwe.get_support() + assert_array_equal(support_fwe, np.array([True, False])) + + def test_select_fdr_regression(): # Test that fdr heuristic actually has low FDR. def single_fdr(alpha, n_informative, random_state): @@ -404,7 +438,7 @@ def single_fdr(alpha, n_informative, random_state): # FDR = E(FP / (TP + FP)) <= alpha false_discovery_rate = np.mean([single_fdr(alpha, n_informative, random_state) for - random_state in range(30)]) + random_state in range(100)]) assert_greater_equal(alpha, false_discovery_rate) # Make sure that the empirical false discovery rate increases @@ -485,6 +519,21 @@ def test_tied_pvalues(): assert_not_in(9998, Xt) +def test_scorefunc_multilabel(): + # Test whether k-best and percentiles works with multilabels with chi2. + + X = np.array([[10000, 9999, 0], [100, 9999, 0], [1000, 99, 0]]) + y = [[1, 1], [0, 1], [1, 0]] + + Xt = SelectKBest(chi2, k=2).fit_transform(X, y) + assert_equal(Xt.shape, (3, 2)) + assert_not_in(0, Xt) + + Xt = SelectPercentile(chi2, percentile=67).fit_transform(X, y) + assert_equal(Xt.shape, (3, 2)) + assert_not_in(0, Xt) + + def test_tied_scores(): # Test for stable sorting in k-best with tied scores. X_train = np.array([[0, 0, 0], [1, 1, 1]]) diff --git a/sklearn/feature_selection/tests/test_from_model.py b/sklearn/feature_selection/tests/test_from_model.py index fe170f09ad80c..a4789de0976bb 100644 --- a/sklearn/feature_selection/tests/test_from_model.py +++ b/sklearn/feature_selection/tests/test_from_model.py @@ -17,6 +17,7 @@ from sklearn.feature_selection import SelectFromModel from sklearn.ensemble import RandomForestClassifier from sklearn.linear_model import PassiveAggressiveClassifier +from sklearn.utils.fixes import norm iris = datasets.load_iris() data, y = iris.data, iris.target @@ -102,6 +103,31 @@ def test_feature_importances(): assert_array_equal(X_new, X[:, mask]) +@skip_if_32bit +def test_feature_importances_2d_coef(): + X, y = datasets.make_classification( + n_samples=1000, n_features=10, n_informative=3, n_redundant=0, + n_repeated=0, shuffle=False, random_state=0, n_classes=4) + + est = LogisticRegression() + for threshold, func in zip(["mean", "median"], [np.mean, np.median]): + for order in [1, 2, np.inf]: + # Fit SelectFromModel a multi-class problem + transformer = SelectFromModel(estimator=LogisticRegression(), + threshold=threshold, + norm_order=order) + transformer.fit(X, y) + assert_true(hasattr(transformer.estimator_, 'coef_')) + X_new = transformer.transform(X) + assert_less(X_new.shape[1], X.shape[1]) + + # Manually check that the norm is correctly performed + est.fit(X, y) + importances = norm(est.coef_, axis=0, ord=order) + feature_mask = importances > func(importances) + assert_array_equal(X_new, X[:, feature_mask]) + + def test_partial_fit(): est = PassiveAggressiveClassifier(random_state=0, shuffle=False) transformer = SelectFromModel(estimator=est) diff --git a/sklearn/feature_selection/univariate_selection.py b/sklearn/feature_selection/univariate_selection.py index 16002101a37f8..ffee1e369c937 100644 --- a/sklearn/feature_selection/univariate_selection.py +++ b/sklearn/feature_selection/univariate_selection.py @@ -319,7 +319,7 @@ def fit(self, X, y): self : object Returns self. """ - X, y = check_X_y(X, y, ['csr', 'csc']) + X, y = check_X_y(X, y, ['csr', 'csc'], multi_output=True) if not callable(self.score_func): raise TypeError("The score function should be a callable, %s (%s) " @@ -596,8 +596,8 @@ def _get_support_mask(self): n_features = len(self.pvalues_) sv = np.sort(self.pvalues_) - selected = sv[sv <= float(self.alpha) / n_features - * np.arange(n_features)] + selected = sv[sv <= float(self.alpha) / n_features * + np.arange(1, n_features + 1)] if selected.size == 0: return np.zeros_like(self.pvalues_, dtype=bool) return self.pvalues_ <= selected.max() diff --git a/sklearn/gaussian_process/gaussian_process.py b/sklearn/gaussian_process/gaussian_process.py index 9e78ba2ea710c..c521cb5b52f43 100644 --- a/sklearn/gaussian_process/gaussian_process.py +++ b/sklearn/gaussian_process/gaussian_process.py @@ -64,8 +64,9 @@ def l1_cross_distances(X): class GaussianProcess(BaseEstimator, RegressorMixin): """The legacy Gaussian Process model class. - Note that this class was deprecated in version 0.18 and will be - removed in 0.20. Use the GaussianProcessRegressor instead. + .. deprecated:: 0.18 + This class will be removed in 0.20. + Use the :class:`GaussianProcessRegressor` instead. Read more in the :ref:`User Guide `. diff --git a/sklearn/gaussian_process/tests/test_gpr.py b/sklearn/gaussian_process/tests/test_gpr.py index e62a2c1b14d31..6324dcb1838bc 100644 --- a/sklearn/gaussian_process/tests/test_gpr.py +++ b/sklearn/gaussian_process/tests/test_gpr.py @@ -41,8 +41,8 @@ def test_gpr_interpolation(): gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_pred, y_cov = gpr.predict(X, return_cov=True) - assert_true(np.allclose(y_pred, y)) - assert_true(np.allclose(np.diag(y_cov), 0.)) + assert_almost_equal(y_pred, y) + assert_almost_equal(np.diag(y_cov), 0.) def test_lml_improving(): diff --git a/sklearn/grid_search.py b/sklearn/grid_search.py index 0de08ee9e89f0..6c0101a559bcc 100644 --- a/sklearn/grid_search.py +++ b/sklearn/grid_search.py @@ -46,6 +46,10 @@ class ParameterGrid(object): """Grid of parameters with a discrete number of values for each. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.ParameterGrid` instead. + Can be used to iterate over parameter value combinations with the Python built-in function iter. @@ -165,6 +169,10 @@ def __getitem__(self, ind): class ParameterSampler(object): """Generator on parameters sampled from given distributions. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.ParameterSampler` instead. + Non-deterministic iterable over random candidate combinations for hyper- parameter search. If all parameters are presented as a list, sampling without replacement is performed. If at least one parameter @@ -265,6 +273,10 @@ def fit_grid_point(X, y, estimator, parameters, train, test, scorer, verbose, error_score='raise', **fit_params): """Run fit on one set of parameters. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :func:`sklearn.model_selection.fit_grid_point` instead. + Parameters ---------- X : array-like, sparse matrix or list @@ -387,6 +399,10 @@ def __init__(self, estimator, scoring=None, def _estimator_type(self): return self.estimator._estimator_type + @property + def classes_(self): + return self.best_estimator_.classes_ + def score(self, X, y=None): """Returns the score on the given data, if the estimator has been refit. @@ -614,6 +630,10 @@ def _fit(self, X, y, parameter_iterable): class GridSearchCV(BaseSearchCV): """Exhaustive search over specified parameter values for an estimator. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.GridSearchCV` instead. + Important members are fit, predict. GridSearchCV implements a "fit" and a "score" method. @@ -688,7 +708,7 @@ class GridSearchCV(BaseSearchCV): - An iterable yielding train/test splits. For integer/None inputs, if the estimator is a classifier and ``y`` is - either binary or multiclass, + either binary or multiclass, :class:`sklearn.model_selection.StratifiedKFold` is used. In all other cases, :class:`sklearn.model_selection.KFold` is used. @@ -816,6 +836,9 @@ def fit(self, X, y=None): class RandomizedSearchCV(BaseSearchCV): """Randomized search on hyper parameters. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :class:`sklearn.model_selection.RandomizedSearchCV` instead. RandomizedSearchCV implements a "fit" and a "score" method. It also implements "predict", "predict_proba", "decision_function", @@ -900,7 +923,7 @@ class RandomizedSearchCV(BaseSearchCV): - An iterable yielding train/test splits. For integer/None inputs, if the estimator is a classifier and ``y`` is - either binary or multiclass, + either binary or multiclass, :class:`sklearn.model_selection.StratifiedKFold` is used. In all other cases, :class:`sklearn.model_selection.KFold` is used. diff --git a/sklearn/lda.py b/sklearn/lda.py index 8a488ecc7e38e..9c3959b6bc102 100644 --- a/sklearn/lda.py +++ b/sklearn/lda.py @@ -1,6 +1,20 @@ import warnings +from .discriminant_analysis import LinearDiscriminantAnalysis as _LDA + warnings.warn("lda.LDA has been moved to " "discriminant_analysis.LinearDiscriminantAnalysis " "in 0.17 and will be removed in 0.19", DeprecationWarning) -from .discriminant_analysis import LinearDiscriminantAnalysis as LDA + +class LDA(_LDA): + """ + Alias for + :class:`sklearn.discriminant_analysis.LinearDiscriminantAnalysis`. + + .. deprecated:: 0.17 + This class will be removed in 0.19. + Use + :class:`sklearn.discriminant_analysis.LinearDiscriminantAnalysis` + instead. + """ + pass diff --git a/sklearn/learning_curve.py b/sklearn/learning_curve.py index 1642b9a7c4dc0..59d55cad3eb7f 100644 --- a/sklearn/learning_curve.py +++ b/sklearn/learning_curve.py @@ -32,6 +32,10 @@ def learning_curve(estimator, X, y, train_sizes=np.linspace(0.1, 1.0, 5), error_score='raise'): """Learning curve. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :func:`sklearn.model_selection.learning_curve` instead. + Determines cross-validated training and test scores for different training set sizes. @@ -259,6 +263,10 @@ def validation_curve(estimator, X, y, param_name, param_range, cv=None, scoring=None, n_jobs=1, pre_dispatch="all", verbose=0): """Validation curve. + .. deprecated:: 0.18 + This module will be removed in 0.20. + Use :func:`sklearn.model_selection.validation_curve` instead. + Determine training and test scores for varying parameter values. Compute scores for an estimator with different values of a specified diff --git a/sklearn/linear_model/coordinate_descent.py b/sklearn/linear_model/coordinate_descent.py index 5002a8af44408..100c0afdd814c 100644 --- a/sklearn/linear_model/coordinate_descent.py +++ b/sklearn/linear_model/coordinate_descent.py @@ -49,10 +49,10 @@ def _alpha_grid(X, y, Xy=None, l1_ratio=1.0, fit_intercept=True, Xy = np.dot(X.T, y) that can be precomputed. l1_ratio : float - The elastic net mixing parameter, with ``0 <= l1_ratio <= 1``. - For ``l1_ratio = 0`` the penalty is an L2 penalty. ``For - l1_ratio = 1`` it is an L1 penalty. For ``0 < l1_ratio < - 1``, the penalty is a combination of L1 and L2. + The elastic net mixing parameter, with ``0 < l1_ratio <= 1``. + For ``l1_ratio = 0`` the penalty is an L2 penalty. (currently not + supported) ``For l1_ratio = 1`` it is an L1 penalty. For + ``0 < l1_ratio <1``, the penalty is a combination of L1 and L2. eps : float, optional Length of the path. ``eps=1e-3`` means that @@ -77,6 +77,11 @@ def _alpha_grid(X, y, Xy=None, l1_ratio=1.0, fit_intercept=True, copy_X : boolean, optional, default True If ``True``, X will be copied; else, it may be overwritten. """ + if l1_ratio == 0: + raise ValueError("Automatic alpha grid generation is not supported for" + " l1_ratio=0. Please supply a grid by providing " + "your estimator with the appropriate `alphas=` " + "argument.") n_samples = len(y) sparse_center = False @@ -1559,7 +1564,7 @@ class MultiTaskElasticNet(Lasso): i.e. the sum of norm of each row. - Read more in the :ref:`User Guide `. + Read more in the :ref:`User Guide `. Parameters ---------- diff --git a/sklearn/linear_model/omp.py b/sklearn/linear_model/omp.py index a75146c865535..41acb4cd4780d 100644 --- a/sklearn/linear_model/omp.py +++ b/sklearn/linear_model/omp.py @@ -336,7 +336,7 @@ def orthogonal_mp(X, y, n_nonzero_coefs=None, tol=None, precompute=False, Notes ----- - Orthogonal matching pursuit was introduced in G. Mallat, Z. Zhang, + Orthogonal matching pursuit was introduced in S. Mallat, Z. Zhang, Matching pursuits with time-frequency dictionaries, IEEE Transactions on Signal Processing, Vol. 41, No. 12. (December 1993), pp. 3397-3415. (http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf) diff --git a/sklearn/linear_model/tests/test_coordinate_descent.py b/sklearn/linear_model/tests/test_coordinate_descent.py index 6d1286cb23481..fdb11e3bb5aa2 100644 --- a/sklearn/linear_model/tests/test_coordinate_descent.py +++ b/sklearn/linear_model/tests/test_coordinate_descent.py @@ -17,6 +17,7 @@ from sklearn.utils.testing import assert_greater from sklearn.utils.testing import assert_raises from sklearn.utils.testing import assert_raises_regex +from sklearn.utils.testing import assert_raise_message from sklearn.utils.testing import assert_warns from sklearn.utils.testing import assert_warns_message from sklearn.utils.testing import ignore_warnings @@ -712,3 +713,35 @@ def test_enet_float_precision(): assert_array_almost_equal(intercept[np.float32], intercept[np.float64], decimal=4) + + +def test_enet_l1_ratio(): + # Test that an error message is raised if an estimator that + # uses _alpha_grid is called with l1_ratio=0 + msg = ("Automatic alpha grid generation is not supported for l1_ratio=0. " + "Please supply a grid by providing your estimator with the " + "appropriate `alphas=` argument.") + X = np.array([[1, 2, 4, 5, 8], [3, 5, 7, 7, 8]]).T + y = np.array([12, 10, 11, 21, 5]) + + assert_raise_message(ValueError, msg, ElasticNetCV( + l1_ratio=0, random_state=42).fit, X, y) + assert_raise_message(ValueError, msg, MultiTaskElasticNetCV( + l1_ratio=0, random_state=42).fit, X, y[:, None]) + + # Test that l1_ratio=0 is allowed if we supply a grid manually + alphas = [0.1, 10] + estkwds = {'alphas': alphas, 'random_state': 42} + est_desired = ElasticNetCV(l1_ratio=0.00001, **estkwds) + est = ElasticNetCV(l1_ratio=0, **estkwds) + with ignore_warnings(): + est_desired.fit(X, y) + est.fit(X, y) + assert_array_almost_equal(est.coef_, est_desired.coef_, decimal=5) + + est_desired = MultiTaskElasticNetCV(l1_ratio=0.00001, **estkwds) + est = MultiTaskElasticNetCV(l1_ratio=0, **estkwds) + with ignore_warnings(): + est.fit(X, y[:, None]) + est_desired.fit(X, y[:, None]) + assert_array_almost_equal(est.coef_, est_desired.coef_, decimal=5) diff --git a/sklearn/linear_model/tests/test_logistic.py b/sklearn/linear_model/tests/test_logistic.py index cc840335c8288..8d35bb220c958 100644 --- a/sklearn/linear_model/tests/test_logistic.py +++ b/sklearn/linear_model/tests/test_logistic.py @@ -588,24 +588,28 @@ def test_logistic_regression_sample_weights(): # Test that passing sample_weight as ones is the same as # not passing them at all (default None) for solver in ['lbfgs', 'liblinear']: - clf_sw_none = LR(solver=solver, fit_intercept=False) + clf_sw_none = LR(solver=solver, fit_intercept=False, + random_state=42) clf_sw_none.fit(X, y) - clf_sw_ones = LR(solver=solver, fit_intercept=False) + clf_sw_ones = LR(solver=solver, fit_intercept=False, + random_state=42) clf_sw_ones.fit(X, y, sample_weight=np.ones(y.shape[0])) assert_array_almost_equal( clf_sw_none.coef_, clf_sw_ones.coef_, decimal=4) # Test that sample weights work the same with the lbfgs, # newton-cg, and 'sag' solvers - clf_sw_lbfgs = LR(solver='lbfgs', fit_intercept=False) + clf_sw_lbfgs = LR(solver='lbfgs', fit_intercept=False, random_state=42) clf_sw_lbfgs.fit(X, y, sample_weight=sample_weight) - clf_sw_n = LR(solver='newton-cg', fit_intercept=False) + clf_sw_n = LR(solver='newton-cg', fit_intercept=False, random_state=42) clf_sw_n.fit(X, y, sample_weight=sample_weight) - clf_sw_sag = LR(solver='sag', fit_intercept=False, tol=1e-10) + clf_sw_sag = LR(solver='sag', fit_intercept=False, tol=1e-10, + random_state=42) # ignore convergence warning due to small dataset with ignore_warnings(): clf_sw_sag.fit(X, y, sample_weight=sample_weight) - clf_sw_liblinear = LR(solver='liblinear', fit_intercept=False) + clf_sw_liblinear = LR(solver='liblinear', fit_intercept=False, + random_state=42) clf_sw_liblinear.fit(X, y, sample_weight=sample_weight) assert_array_almost_equal( clf_sw_lbfgs.coef_, clf_sw_n.coef_, decimal=4) @@ -619,9 +623,9 @@ def test_logistic_regression_sample_weights(): # to be 2 for all instances of class 2 for solver in ['lbfgs', 'liblinear']: clf_cw_12 = LR(solver=solver, fit_intercept=False, - class_weight={0: 1, 1: 2}) + class_weight={0: 1, 1: 2}, random_state=42) clf_cw_12.fit(X, y) - clf_sw_12 = LR(solver=solver, fit_intercept=False) + clf_sw_12 = LR(solver=solver, fit_intercept=False, random_state=42) clf_sw_12.fit(X, y, sample_weight=sample_weight) assert_array_almost_equal( clf_cw_12.coef_, clf_sw_12.coef_, decimal=4) @@ -630,19 +634,21 @@ def test_logistic_regression_sample_weights(): # since the patched liblinear code is different. clf_cw = LogisticRegression( solver="liblinear", fit_intercept=False, class_weight={0: 1, 1: 2}, - penalty="l1", tol=1e-5) + penalty="l1", tol=1e-5, random_state=42) clf_cw.fit(X, y) clf_sw = LogisticRegression( - solver="liblinear", fit_intercept=False, penalty="l1", tol=1e-5) + solver="liblinear", fit_intercept=False, penalty="l1", tol=1e-5, + random_state=42) clf_sw.fit(X, y, sample_weight) assert_array_almost_equal(clf_cw.coef_, clf_sw.coef_, decimal=4) clf_cw = LogisticRegression( solver="liblinear", fit_intercept=False, class_weight={0: 1, 1: 2}, - penalty="l2", dual=True) + penalty="l2", dual=True, random_state=42) clf_cw.fit(X, y) clf_sw = LogisticRegression( - solver="liblinear", fit_intercept=False, penalty="l2", dual=True) + solver="liblinear", fit_intercept=False, penalty="l2", dual=True, + random_state=42) clf_sw.fit(X, y, sample_weight) assert_array_almost_equal(clf_cw.coef_, clf_sw.coef_, decimal=4) diff --git a/sklearn/manifold/locally_linear.py b/sklearn/manifold/locally_linear.py index a1940333e5094..f5a383d58a350 100644 --- a/sklearn/manifold/locally_linear.py +++ b/sklearn/manifold/locally_linear.py @@ -10,6 +10,7 @@ from ..base import BaseEstimator, TransformerMixin from ..utils import check_random_state, check_array from ..utils.arpack import eigsh +from ..utils.extmath import stable_cumsum from ..utils.validation import check_is_fitted from ..utils.validation import FLOAT_DTYPES from ..neighbors import NearestNeighbors @@ -420,7 +421,7 @@ def locally_linear_embedding( # this is the size of the largest set of eigenvalues # such that Sum[v; v in set]/Sum[v; v not in set] < eta s_range = np.zeros(N, dtype=int) - evals_cumsum = np.cumsum(evals, 1) + evals_cumsum = stable_cumsum(evals, 1) eta_range = evals_cumsum[:, -1:] / evals_cumsum[:, :-1] - 1 for i in range(N): s_range[i] = np.searchsorted(eta_range[i, ::-1], eta) diff --git a/sklearn/metrics/pairwise.py b/sklearn/metrics/pairwise.py index e8f5090cba64b..9ca8a443ca359 100644 --- a/sklearn/metrics/pairwise.py +++ b/sklearn/metrics/pairwise.py @@ -24,7 +24,7 @@ from ..preprocessing import normalize from ..externals.joblib import Parallel from ..externals.joblib import delayed -from ..externals.joblib.parallel import cpu_count +from ..externals.joblib import cpu_count from .pairwise_fast import _chi2_kernel_fast, _sparse_manhattan @@ -570,6 +570,11 @@ def cosine_distances(X, Y=None): S = cosine_similarity(X, Y) S *= -1 S += 1 + np.clip(S, 0, 2, out=S) + if X is Y or Y is None: + # Ensure that distances between vectors and themselves are set to 0.0. + # This may not be the case due to floating point rounding errors. + S[np.diag_indices_from(S)] = 0.0 return S diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index af3a02d6f33f9..beeae240b4ea1 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -438,7 +438,18 @@ def r2_score(y_true, y_pred, >>> y_pred = [[0, 2], [-1, 2], [8, -5]] >>> r2_score(y_true, y_pred, multioutput='variance_weighted') # doctest: +ELLIPSIS 0.938... - + >>> y_true = [1,2,3] + >>> y_pred = [1,2,3] + >>> r2_score(y_true, y_pred) + 1.0 + >>> y_true = [1,2,3] + >>> y_pred = [2,2,2] + >>> r2_score(y_true, y_pred) + 0.0 + >>> y_true = [1,2,3] + >>> y_pred = [3,2,1] + >>> r2_score(y_true, y_pred) + -3.0 """ y_type, y_true, y_pred, multioutput = _check_reg_targets( y_true, y_pred, multioutput) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index fa4c7e8d3124b..0b2f941e17039 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -22,6 +22,7 @@ from sklearn.utils.testing import assert_raise_message from sklearn.utils.testing import assert_true from sklearn.utils.testing import ignore_warnings +from sklearn.utils.testing import _named_check from sklearn.metrics import accuracy_score from sklearn.metrics import average_precision_score @@ -891,8 +892,8 @@ def test_averaging_multiclass(n_samples=50, n_classes=3): y_pred_binarize = lb.transform(y_pred) for name in METRICS_WITH_AVERAGING: - yield (check_averaging, name, y_true, y_true_binarize, y_pred, - y_pred_binarize, y_score) + yield (_named_check(check_averaging, name), name, y_true, + y_true_binarize, y_pred, y_pred_binarize, y_score) def test_averaging_multilabel(n_classes=5, n_samples=40): @@ -906,8 +907,8 @@ def test_averaging_multilabel(n_classes=5, n_samples=40): y_pred_binarize = y_pred for name in METRICS_WITH_AVERAGING + THRESHOLDED_METRICS_WITH_AVERAGING: - yield (check_averaging, name, y_true, y_true_binarize, y_pred, - y_pred_binarize, y_score) + yield (_named_check(check_averaging, name), name, y_true, + y_true_binarize, y_pred, y_pred_binarize, y_score) def test_averaging_multilabel_all_zeroes(): @@ -918,8 +919,8 @@ def test_averaging_multilabel_all_zeroes(): y_pred_binarize = y_pred for name in METRICS_WITH_AVERAGING: - yield (check_averaging, name, y_true, y_true_binarize, y_pred, - y_pred_binarize, y_score) + yield (_named_check(check_averaging, name), name, y_true, + y_true_binarize, y_pred, y_pred_binarize, y_score) # Test _average_binary_score for weight.sum() == 0 binary_metric = (lambda y_true, y_score, average="macro": @@ -937,8 +938,8 @@ def test_averaging_multilabel_all_ones(): y_pred_binarize = y_pred for name in METRICS_WITH_AVERAGING: - yield (check_averaging, name, y_true, y_true_binarize, y_pred, - y_pred_binarize, y_score) + yield (_named_check(check_averaging, name), name, y_true, + y_true_binarize, y_pred, y_pred_binarize, y_score) @ignore_warnings @@ -1025,9 +1026,11 @@ def test_sample_weight_invariance(n_samples=50): continue metric = ALL_METRICS[name] if name in THRESHOLDED_METRICS: - yield check_sample_weight_invariance, name, metric, y_true, y_score + yield _named_check(check_sample_weight_invariance, name), name,\ + metric, y_true, y_score else: - yield check_sample_weight_invariance, name, metric, y_true, y_pred + yield _named_check(check_sample_weight_invariance, name), name,\ + metric, y_true, y_pred # multiclass random_state = check_random_state(0) @@ -1040,9 +1043,11 @@ def test_sample_weight_invariance(n_samples=50): continue metric = ALL_METRICS[name] if name in THRESHOLDED_METRICS: - yield check_sample_weight_invariance, name, metric, y_true, y_score + yield _named_check(check_sample_weight_invariance, name), name,\ + metric, y_true, y_score else: - yield check_sample_weight_invariance, name, metric, y_true, y_pred + yield _named_check(check_sample_weight_invariance, name), name,\ + metric, y_true, y_pred # multilabel indicator _, ya = make_multilabel_classification(n_features=1, n_classes=20, @@ -1062,11 +1067,11 @@ def test_sample_weight_invariance(n_samples=50): metric = ALL_METRICS[name] if name in THRESHOLDED_METRICS: - yield (check_sample_weight_invariance, name, metric, y_true, - y_score) + yield (_named_check(check_sample_weight_invariance, name), name, + metric, y_true, y_score) else: - yield (check_sample_weight_invariance, name, metric, y_true, - y_pred) + yield (_named_check(check_sample_weight_invariance, name), name, + metric, y_true, y_pred) @ignore_warnings diff --git a/sklearn/metrics/tests/test_pairwise.py b/sklearn/metrics/tests/test_pairwise.py index 01863166953d6..64ca3a1902f90 100644 --- a/sklearn/metrics/tests/test_pairwise.py +++ b/sklearn/metrics/tests/test_pairwise.py @@ -407,6 +407,36 @@ def test_euclidean_distances(): assert_greater(np.max(np.abs(wrong_D - D1)), .01) +def test_cosine_distances(): + # Check the pairwise Cosine distances computation + rng = np.random.RandomState(1337) + x = np.abs(rng.rand(910)) + XA = np.vstack([x, x]) + D = cosine_distances(XA) + assert_array_almost_equal(D, [[0., 0.], [0., 0.]]) + # check that all elements are in [0, 2] + assert_true(np.all(D >= 0.)) + assert_true(np.all(D <= 2.)) + # check that diagonal elements are equal to 0 + assert_array_equal(D[np.diag_indices_from(D)], [0., 0.]) + + XB = np.vstack([x, -x]) + D2 = cosine_distances(XB) + # check that all elements are in [0, 2] + assert_true(np.all(D2 >= 0.)) + assert_true(np.all(D2 <= 2.)) + # check that diagonal elements are equal to 0 and non diagonal to 2 + assert_array_equal(D2, [[0., 2.], [2., 0.]]) + + # check large random matrix + X = np.abs(rng.rand(1000, 5000)) + D = cosine_distances(X) + # check that diagonal elements are equal to 0 + assert_array_almost_equal(D[np.diag_indices_from(D)], [0.] * D.shape[0]) + assert_true(np.all(D >= 0.)) + assert_true(np.all(D <= 2.)) + + # Paired distances def test_paired_euclidean_distances(): diff --git a/sklearn/mixture/base.py b/sklearn/mixture/base.py index 4cec169fd0d2f..ad78183bb16d1 100644 --- a/sklearn/mixture/base.py +++ b/sklearn/mixture/base.py @@ -250,8 +250,8 @@ def _e_step(self, X): Returns ------- - log_prob_norm : array, shape (n_samples,) - Logarithm of the probability of each sample in X. + log_prob_norm : float + Mean of the logarithms of the probabilities of each sample in X log_responsibility : array, shape (n_samples, n_components) Logarithm of the posterior probabilities (or responsibilities) of @@ -385,7 +385,7 @@ def sample(self, n_samples=1): _, n_features = self.means_.shape rng = check_random_state(self.random_state) - n_samples_comp = np.round(self.weights_ * n_samples).astype(int) + n_samples_comp = rng.multinomial(n_samples, self.weights_) if self.covariance_type == 'full': X = np.vstack([ diff --git a/sklearn/mixture/dpgmm.py b/sklearn/mixture/dpgmm.py index 0b7f11affe3ca..bbbf0b9e217f2 100644 --- a/sklearn/mixture/dpgmm.py +++ b/sklearn/mixture/dpgmm.py @@ -24,7 +24,7 @@ from ..externals.six.moves import xrange from ..utils import check_random_state, check_array, deprecated -from ..utils.extmath import logsumexp, pinvh, squared_norm +from ..utils.extmath import logsumexp, pinvh, squared_norm, stable_cumsum from ..utils.validation import check_is_fitted from .. import cluster from .gmm import _GMMBase @@ -462,7 +462,7 @@ def _bound_proportions(self, z): dg1 = digamma(self.gamma_.T[1]) - dg12 dg2 = digamma(self.gamma_.T[2]) - dg12 - cz = np.cumsum(z[:, ::-1], axis=-1)[:, -2::-1] + cz = stable_cumsum(z[:, ::-1], axis=-1)[:, -2::-1] logprior = np.sum(cz * dg2[:-1]) + np.sum(z * dg1) del cz # Save memory z_non_zeros = z[z > np.finfo(np.float32).eps] @@ -629,6 +629,16 @@ def _fit(self, X, y=None): "instead. DPGMM is deprecated in 0.18 and will be " "removed in 0.20.") class DPGMM(_DPGMMBase): + """Dirichlet Process Gaussian Mixture Models + + .. deprecated:: 0.18 + This class will be removed in 0.20. + Use :class:`sklearn.mixture.BayesianGaussianMixture` with + parameter ``weight_concentration_prior_type='dirichlet_process'`` + instead. + + """ + def __init__(self, n_components=1, covariance_type='diag', alpha=1.0, random_state=None, tol=1e-3, verbose=0, min_covar=None, n_iter=10, params='wmc', init_params='wmc'): @@ -647,6 +657,11 @@ def __init__(self, n_components=1, covariance_type='diag', alpha=1.0, class VBGMM(_DPGMMBase): """Variational Inference for the Gaussian Mixture Model + .. deprecated:: 0.18 + This class will be removed in 0.20. + Use :class:`sklearn.mixture.BayesianGaussianMixture` with parameter + ``weight_concentration_prior_type='dirichlet_distribution'`` instead. + Variational inference for a Gaussian mixture model probability distribution. This class allows for easy and efficient inference of an approximate posterior distribution over the parameters of a diff --git a/sklearn/mixture/gmm.py b/sklearn/mixture/gmm.py index d1a4eb818a6c0..c00b138c6b13e 100644 --- a/sklearn/mixture/gmm.py +++ b/sklearn/mixture/gmm.py @@ -662,6 +662,15 @@ def aic(self, X): @deprecated("The class GMM is deprecated in 0.18 and will be " " removed in 0.20. Use class GaussianMixture instead.") class GMM(_GMMBase): + """ + Legacy Gaussian Mixture Model + + .. deprecated:: 0.18 + This class will be removed in 0.20. + Use :class:`sklearn.mixture.GaussianMixture` instead. + + """ + def __init__(self, n_components=1, covariance_type='diag', random_state=None, tol=1e-3, min_covar=1e-3, n_iter=100, n_init=1, params='wmc', init_params='wmc', diff --git a/sklearn/mixture/tests/test_gaussian_mixture.py b/sklearn/mixture/tests/test_gaussian_mixture.py index 43d61e010a3a0..3aba57494cb94 100644 --- a/sklearn/mixture/tests/test_gaussian_mixture.py +++ b/sklearn/mixture/tests/test_gaussian_mixture.py @@ -918,7 +918,7 @@ def test_property(): def test_sample(): rng = np.random.RandomState(0) - rand_data = RandomData(rng, scale=7) + rand_data = RandomData(rng, scale=7, n_components=3) n_features, n_components = rand_data.n_features, rand_data.n_components for covar_type in COVARIANCE_TYPE: @@ -935,8 +935,10 @@ def test_sample(): gmm.sample, 0) # Just to make sure the class samples correctly - X_s, y_s = gmm.sample(20000) - for k in range(n_features): + n_samples = 20000 + X_s, y_s = gmm.sample(n_samples) + + for k in range(n_components): if covar_type == 'full': assert_array_almost_equal(gmm.covariances_[k], np.cov(X_s[y_s == k].T), decimal=1) @@ -953,9 +955,17 @@ def test_sample(): decimal=1) means_s = np.array([np.mean(X_s[y_s == k], 0) - for k in range(n_features)]) + for k in range(n_components)]) assert_array_almost_equal(gmm.means_, means_s, decimal=1) + # Check shapes of sampled data, see + # https://github.com/scikit-learn/scikit-learn/issues/7701 + assert_equal(X_s.shape, (n_samples, n_features)) + + for sample_size in range(1, 100): + X_s, _ = gmm.sample(sample_size) + assert_equal(X_s.shape, (sample_size, n_features)) + @ignore_warnings(category=ConvergenceWarning) def test_init(): diff --git a/sklearn/model_selection/_split.py b/sklearn/model_selection/_split.py index cf109e621626b..0064830c9a952 100644 --- a/sklearn/model_selection/_split.py +++ b/sklearn/model_selection/_split.py @@ -773,6 +773,10 @@ def _iter_test_masks(self, X, y, groups): # We make a copy of groups to avoid side-effects during iteration groups = np.array(groups, copy=True) unique_groups = np.unique(groups) + if len(unique_groups) <= 1: + raise ValueError( + "The groups parameter contains fewer than 2 unique groups " + "(%s). LeaveOneGroupOut expects at least 2." % unique_groups) for i in unique_groups: yield groups == i @@ -862,6 +866,12 @@ def _iter_test_masks(self, X, y, groups): raise ValueError("The groups parameter should not be None") groups = np.array(groups, copy=True) unique_groups = np.unique(groups) + if self.n_groups >= len(unique_groups): + raise ValueError( + "The groups parameter contains fewer than (or equal to) " + "n_groups (%d) numbers of unique groups (%s). LeavePGroupsOut " + "expects that at least n_groups + 1 (%d) unique groups be " + "present" % (self.n_groups, unique_groups, self.n_groups + 1)) combi = combinations(range(len(unique_groups)), self.n_groups) for indices in combi: test_index = np.zeros(_num_samples(X), dtype=np.bool) diff --git a/sklearn/model_selection/_validation.py b/sklearn/model_selection/_validation.py index 9745cb9decf73..b8546d804eb24 100644 --- a/sklearn/model_selection/_validation.py +++ b/sklearn/model_selection/_validation.py @@ -265,7 +265,8 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, if verbose > 2: msg += ", score=%f" % test_score if verbose > 1: - end_msg = "%s -%s" % (msg, logger.short_format_time(score_time)) + total_time = score_time + fit_time + end_msg = "%s, total=%s" % (msg, logger.short_format_time(total_time)) print("[CV] %s %s" % ((64 - len(end_msg)) * '.', end_msg)) ret = [train_score, test_score] if return_train_score else [test_score] @@ -642,7 +643,8 @@ def _shuffle(y, groups, random_state): def learning_curve(estimator, X, y, groups=None, train_sizes=np.linspace(0.1, 1.0, 5), cv=None, scoring=None, exploit_incremental_learning=False, n_jobs=1, - pre_dispatch="all", verbose=0): + pre_dispatch="all", verbose=0, shuffle=False, + random_state=None): """Learning curve. Determines cross-validated training and test scores for different training @@ -718,7 +720,14 @@ def learning_curve(estimator, X, y, groups=None, verbose : integer, optional Controls the verbosity: the higher, the more messages. - Returns + shuffle : boolean, optional + Whether to shuffle training data before taking prefixes of it + based on``train_sizes``. + + random_state : None, int or RandomState + When shuffle=True, pseudo-random number generator state used for + shuffling. If None, use default numpy RNG for shuffling. + ------- train_sizes_abs : array, shape = (n_unique_ticks,), dtype int Numbers of training examples that has been used to generate the @@ -759,17 +768,27 @@ def learning_curve(estimator, X, y, groups=None, parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch, verbose=verbose) + + if shuffle: + rng = check_random_state(random_state) + cv_iter = ((rng.permutation(train), test) for train, test in cv_iter) + if exploit_incremental_learning: classes = np.unique(y) if is_classifier(estimator) else None out = parallel(delayed(_incremental_fit_estimator)( - clone(estimator), X, y, classes, train, test, train_sizes_abs, - scorer, verbose) for train, test in cv.split(X, y, groups)) + clone(estimator), X, y, classes, train, + test, train_sizes_abs, scorer, verbose) + for train, test in cv_iter) else: + train_test_proportions = [] + for train, test in cv_iter: + for n_train_samples in train_sizes_abs: + train_test_proportions.append((train[:n_train_samples], test)) + out = parallel(delayed(_fit_and_score)( - clone(estimator), X, y, scorer, train[:n_train_samples], test, + clone(estimator), X, y, scorer, train, test, verbose, parameters=None, fit_params=None, return_train_score=True) - for train, test in cv_iter - for n_train_samples in train_sizes_abs) + for train, test in train_test_proportions) out = np.array(out) n_cv_folds = out.shape[0] // n_unique_ticks out = out.reshape(n_cv_folds, n_unique_ticks, 2) diff --git a/sklearn/model_selection/tests/test_split.py b/sklearn/model_selection/tests/test_split.py index 4dcd8f55038d8..b547ac6415563 100644 --- a/sklearn/model_selection/tests/test_split.py +++ b/sklearn/model_selection/tests/test_split.py @@ -724,6 +724,31 @@ def test_leave_group_out_changing_groups(): assert_equal(3, LeaveOneGroupOut().get_n_splits(X, y, groups)) +def test_leave_one_p_group_out_error_on_fewer_number_of_groups(): + X = y = groups = np.ones(0) + msg = ("The groups parameter contains fewer than 2 unique groups ([]). " + "LeaveOneGroupOut expects at least 2.") + assert_raise_message(ValueError, msg, next, + LeaveOneGroupOut().split(X, y, groups)) + X = y = groups = np.ones(1) + msg = ("The groups parameter contains fewer than 2 unique groups ([ 1.]). " + "LeaveOneGroupOut expects at least 2.") + assert_raise_message(ValueError, msg, next, + LeaveOneGroupOut().split(X, y, groups)) + X = y = groups = np.ones(1) + msg = ("The groups parameter contains fewer than (or equal to) n_groups " + "(3) numbers of unique groups ([ 1.]). LeavePGroupsOut expects " + "that at least n_groups + 1 (4) unique groups be present") + assert_raise_message(ValueError, msg, next, + LeavePGroupsOut(n_groups=3).split(X, y, groups)) + X = y = groups = np.arange(3) + msg = ("The groups parameter contains fewer than (or equal to) n_groups " + "(3) numbers of unique groups ([0 1 2]). LeavePGroupsOut expects " + "that at least n_groups + 1 (4) unique groups be present") + assert_raise_message(ValueError, msg, next, + LeavePGroupsOut(n_groups=3).split(X, y, groups)) + + def test_train_test_split_errors(): assert_raises(ValueError, train_test_split) assert_raises(ValueError, train_test_split, range(3), train_size=1.1) diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py index 4f45a079fbd7f..26af0f76e690e 100644 --- a/sklearn/model_selection/tests/test_validation.py +++ b/sklearn/model_selection/tests/test_validation.py @@ -560,18 +560,20 @@ def test_learning_curve(): n_redundant=0, n_classes=2, n_clusters_per_class=1, random_state=0) estimator = MockImprovingEstimator(20) - with warnings.catch_warnings(record=True) as w: - train_sizes, train_scores, test_scores = learning_curve( - estimator, X, y, cv=3, train_sizes=np.linspace(0.1, 1.0, 10)) - if len(w) > 0: - raise RuntimeError("Unexpected warning: %r" % w[0].message) - assert_equal(train_scores.shape, (10, 3)) - assert_equal(test_scores.shape, (10, 3)) - assert_array_equal(train_sizes, np.linspace(2, 20, 10)) - assert_array_almost_equal(train_scores.mean(axis=1), - np.linspace(1.9, 1.0, 10)) - assert_array_almost_equal(test_scores.mean(axis=1), - np.linspace(0.1, 1.0, 10)) + for shuffle_train in [False, True]: + with warnings.catch_warnings(record=True) as w: + train_sizes, train_scores, test_scores = learning_curve( + estimator, X, y, cv=3, train_sizes=np.linspace(0.1, 1.0, 10), + shuffle=shuffle_train) + if len(w) > 0: + raise RuntimeError("Unexpected warning: %r" % w[0].message) + assert_equal(train_scores.shape, (10, 3)) + assert_equal(test_scores.shape, (10, 3)) + assert_array_equal(train_sizes, np.linspace(2, 20, 10)) + assert_array_almost_equal(train_scores.mean(axis=1), + np.linspace(1.9, 1.0, 10)) + assert_array_almost_equal(test_scores.mean(axis=1), + np.linspace(0.1, 1.0, 10)) def test_learning_curve_unsupervised(): @@ -622,14 +624,15 @@ def test_learning_curve_incremental_learning(): n_redundant=0, n_classes=2, n_clusters_per_class=1, random_state=0) estimator = MockIncrementalImprovingEstimator(20) - train_sizes, train_scores, test_scores = learning_curve( - estimator, X, y, cv=3, exploit_incremental_learning=True, - train_sizes=np.linspace(0.1, 1.0, 10)) - assert_array_equal(train_sizes, np.linspace(2, 20, 10)) - assert_array_almost_equal(train_scores.mean(axis=1), - np.linspace(1.9, 1.0, 10)) - assert_array_almost_equal(test_scores.mean(axis=1), - np.linspace(0.1, 1.0, 10)) + for shuffle_train in [False, True]: + train_sizes, train_scores, test_scores = learning_curve( + estimator, X, y, cv=3, exploit_incremental_learning=True, + train_sizes=np.linspace(0.1, 1.0, 10), shuffle=shuffle_train) + assert_array_equal(train_sizes, np.linspace(2, 20, 10)) + assert_array_almost_equal(train_scores.mean(axis=1), + np.linspace(1.9, 1.0, 10)) + assert_array_almost_equal(test_scores.mean(axis=1), + np.linspace(0.1, 1.0, 10)) def test_learning_curve_incremental_learning_unsupervised(): @@ -713,6 +716,39 @@ def test_learning_curve_with_boolean_indices(): np.linspace(0.1, 1.0, 10)) +def test_learning_curve_with_shuffle(): + # Following test case was designed this way to verify the code + # changes made in pull request: #7506. + X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [11, 12], [13, 14], [15, 16], + [17, 18], [19, 20], [7, 8], [9, 10], [11, 12], [13, 14], + [15, 16], [17, 18]]) + y = np.array([1, 1, 1, 2, 3, 4, 1, 1, 2, 3, 4, 1, 2, 3, 4]) + groups = np.array([1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 4, 4, 4, 4]) + # Splits on these groups fail without shuffle as the first iteration + # of the learning curve doesn't contain label 4 in the training set. + estimator = PassiveAggressiveClassifier(shuffle=False) + + cv = GroupKFold(n_splits=2) + train_sizes_batch, train_scores_batch, test_scores_batch = learning_curve( + estimator, X, y, cv=cv, n_jobs=1, train_sizes=np.linspace(0.3, 1.0, 3), + groups=groups, shuffle=True, random_state=2) + assert_array_almost_equal(train_scores_batch.mean(axis=1), + np.array([0.75, 0.3, 0.36111111])) + assert_array_almost_equal(test_scores_batch.mean(axis=1), + np.array([0.36111111, 0.25, 0.25])) + assert_raises(ValueError, learning_curve, estimator, X, y, cv=cv, n_jobs=1, + train_sizes=np.linspace(0.3, 1.0, 3), groups=groups) + + train_sizes_inc, train_scores_inc, test_scores_inc = learning_curve( + estimator, X, y, cv=cv, n_jobs=1, train_sizes=np.linspace(0.3, 1.0, 3), + groups=groups, shuffle=True, random_state=2, + exploit_incremental_learning=True) + assert_array_almost_equal(train_scores_inc.mean(axis=1), + train_scores_batch.mean(axis=1)) + assert_array_almost_equal(test_scores_inc.mean(axis=1), + test_scores_batch.mean(axis=1)) + + def test_validation_curve(): X, y = make_classification(n_samples=2, n_features=1, n_informative=1, n_redundant=0, n_classes=2, diff --git a/sklearn/neighbors/__init__.py b/sklearn/neighbors/__init__.py index 14f62a4b6ee89..852b0a5fe32f6 100644 --- a/sklearn/neighbors/__init__.py +++ b/sklearn/neighbors/__init__.py @@ -13,6 +13,7 @@ from .nearest_centroid import NearestCentroid from .kde import KernelDensity from .approximate import LSHForest +from .lof import LocalOutlierFactor __all__ = ['BallTree', 'DistanceMetric', @@ -26,4 +27,5 @@ 'kneighbors_graph', 'radius_neighbors_graph', 'KernelDensity', - 'LSHForest'] + 'LSHForest', + 'LocalOutlierFactor'] diff --git a/sklearn/neighbors/classification.py b/sklearn/neighbors/classification.py index 66da7ff94fc8e..07c8fa320102f 100644 --- a/sklearn/neighbors/classification.py +++ b/sklearn/neighbors/classification.py @@ -29,7 +29,7 @@ class KNeighborsClassifier(NeighborsBase, KNeighborsMixin, Parameters ---------- n_neighbors : int, optional (default = 5) - Number of neighbors to use by default for :meth:`k_neighbors` queries. + Number of neighbors to use by default for :meth:`kneighbors` queries. weights : str or callable, optional (default = 'uniform') weight function used in prediction. Possible values: diff --git a/sklearn/neighbors/lof.py b/sklearn/neighbors/lof.py new file mode 100644 index 0000000000000..eb48f687e1c08 --- /dev/null +++ b/sklearn/neighbors/lof.py @@ -0,0 +1,298 @@ +# Authors: Nicolas Goix +# Alexandre Gramfort +# License: BSD 3 clause + +import numpy as np +from warnings import warn +from scipy.stats import scoreatpercentile + +from .base import NeighborsBase +from .base import KNeighborsMixin +from .base import UnsupervisedMixin + +from ..utils.validation import check_is_fitted +from ..utils import check_array + +__all__ = ["LocalOutlierFactor"] + + +class LocalOutlierFactor(NeighborsBase, KNeighborsMixin, UnsupervisedMixin): + """Unsupervised Outlier Detection using Local Outlier Factor (LOF) + + The anomaly score of each sample is called Local Outlier Factor. + It measures the local deviation of density of a given sample with + respect to its neighbors. + It is local in that the anomaly score depends on how isolated the object + is with respect to the surrounding neighborhood. + More precisely, locality is given by k-nearest neighbors, whose distance + is used to estimate the local density. + By comparing the local density of a sample to the local densities of + its neighbors, one can identify samples that have a substantially lower + density than their neighbors. These are considered outliers. + + Parameters + ---------- + n_neighbors : int, optional (default=20) + Number of neighbors to use by default for :meth:`kneighbors` queries. + If n_neighbors is larger than the number of samples provided, + all samples will be used. + + algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional + Algorithm used to compute the nearest neighbors: + + - 'ball_tree' will use :class:`BallTree` + - 'kd_tree' will use :class:`KDTree` + - 'brute' will use a brute-force search. + - 'auto' will attempt to decide the most appropriate algorithm + based on the values passed to :meth:`fit` method. + + Note: fitting on sparse input will override the setting of + this parameter, using brute force. + + leaf_size : int, optional (default=30) + Leaf size passed to :class:`BallTree` or :class:`KDTree`. This can + affect the speed of the construction and query, as well as the memory + required to store the tree. The optimal value depends on the + nature of the problem. + + p : integer, optional (default=2) + Parameter for the Minkowski metric from + :ref:`sklearn.metrics.pairwise.pairwise_distances`. When p = 1, this is + equivalent to using manhattan_distance (l1), and euclidean_distance + (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. + + metric : string or callable, default 'minkowski' + metric used for the distance computation. Any metric from scikit-learn + or scipy.spatial.distance can be used. + + If 'precomputed', the training input X is expected to be a distance + matrix. + + If metric is a callable function, it is called on each + pair of instances (rows) and the resulting value recorded. The callable + should take two arrays as input and return one value indicating the + distance between them. This works for Scipy's metrics, but is less + efficient than passing the metric name as a string. + + Valid values for metric are: + + - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2', + 'manhattan'] + + - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev', + 'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski', + 'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto', + 'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath', + 'sqeuclidean', 'yule'] + + See the documentation for scipy.spatial.distance for details on these + metrics: + http://docs.scipy.org/doc/scipy/reference/spatial.distance.html + + metric_params : dict, optional (default=None) + Additional keyword arguments for the metric function. + + contamination : float in (0., 0.5), optional (default=0.1) + The amount of contamination of the data set, i.e. the proportion + of outliers in the data set. When fitting this is used to define the + threshold on the decision function. + + n_jobs : int, optional (default=1) + The number of parallel jobs to run for neighbors search. + If ``-1``, then the number of jobs is set to the number of CPU cores. + Affects only :meth:`kneighbors` and :meth:`kneighbors_graph` methods. + + + Attributes + ---------- + negative_outlier_factor_ : numpy array, shape (n_samples,) + The opposite LOF of the training samples. The lower, the more normal. + Inliers tend to have a LOF score close to 1, while outliers tend + to have a larger LOF score. + + The local outlier factor (LOF) of a sample captures its + supposed 'degree of abnormality'. + It is the average of the ratio of the local reachability density of + a sample and those of its k-nearest neighbors. + + n_neighbors_ : integer + The actual number of neighbors used for :meth:`kneighbors` queries. + + References + ---------- + .. [1] Breunig, M. M., Kriegel, H. P., Ng, R. T., & Sander, J. (2000, May). + LOF: identifying density-based local outliers. In ACM sigmod record. + """ + def __init__(self, n_neighbors=20, algorithm='auto', leaf_size=30, + metric='minkowski', p=2, metric_params=None, + contamination=0.1, n_jobs=1): + self._init_params(n_neighbors=n_neighbors, + algorithm=algorithm, + leaf_size=leaf_size, metric=metric, p=p, + metric_params=metric_params, n_jobs=n_jobs) + + self.contamination = contamination + + def fit_predict(self, X, y=None): + """"Fits the model to the training set X and returns the labels + (1 inlier, -1 outlier) on the training set according to the LOF score + and the contamination parameter. + + + Parameters + ---------- + X : array-like, shape (n_samples, n_features), default=None + The query sample or samples to compute the Local Outlier Factor + w.r.t. to the training samples. + + Returns + ------- + is_inlier : array, shape (n_samples,) + Returns 1 for anomalies/outliers and -1 for inliers. + """ + + return self.fit(X)._predict() + + def fit(self, X, y=None): + """Fit the model using X as training data. + + Parameters + ---------- + X : {array-like, sparse matrix, BallTree, KDTree} + Training data. If array or matrix, shape [n_samples, n_features], + or [n_samples, n_samples] if metric='precomputed'. + + Returns + ------- + self : object + Returns self. + """ + if not (0. < self.contamination <= .5): + raise ValueError("contamination must be in (0, 0.5]") + + super(LocalOutlierFactor, self).fit(X) + + n_samples = self._fit_X.shape[0] + if self.n_neighbors > n_samples: + warn("n_neighbors (%s) is greater than the " + "total number of samples (%s). n_neighbors " + "will be set to (n_samples - 1) for estimation." + % (self.n_neighbors, n_samples)) + self.n_neighbors_ = max(1, min(self.n_neighbors, n_samples - 1)) + + self._distances_fit_X_, _neighbors_indices_fit_X_ = ( + self.kneighbors(None, n_neighbors=self.n_neighbors_)) + + self._lrd = self._local_reachability_density( + self._distances_fit_X_, _neighbors_indices_fit_X_) + + # Compute lof score over training samples to define threshold_: + lrd_ratios_array = (self._lrd[_neighbors_indices_fit_X_] / + self._lrd[:, np.newaxis]) + + self.negative_outlier_factor_ = -np.mean(lrd_ratios_array, axis=1) + + self.threshold_ = -scoreatpercentile( + -self.negative_outlier_factor_, 100. * (1. - self.contamination)) + + return self + + def _predict(self, X=None): + """Predict the labels (1 inlier, -1 outlier) of X according to LOF. + + If X is None, returns the same as fit_predict(X_train). + This method allows to generalize prediction to new observations (not + in the training set). As LOF originally does not deal with new data, + this method is kept private. + + Parameters + ---------- + X : array-like, shape (n_samples, n_features), default=None + The query sample or samples to compute the Local Outlier Factor + w.r.t. to the training samples. If None, makes prediction on the + training data without considering them as their own neighbors. + + Returns + ------- + is_inlier : array, shape (n_samples,) + Returns -1 for anomalies/outliers and +1 for inliers. + """ + check_is_fitted(self, ["threshold_", "negative_outlier_factor_", + "n_neighbors_", "_distances_fit_X_"]) + + if X is not None: + X = check_array(X, accept_sparse='csr') + is_inlier = np.ones(X.shape[0], dtype=int) + is_inlier[self._decision_function(X) <= self.threshold_] = -1 + else: + is_inlier = np.ones(self._fit_X.shape[0], dtype=int) + is_inlier[self.negative_outlier_factor_ <= self.threshold_] = -1 + + return is_inlier + + def _decision_function(self, X): + """Opposite of the Local Outlier Factor of X (as bigger is better, + i.e. large values correspond to inliers). + + The argument X is supposed to contain *new data*: if X contains a + point from training, it consider the later in its own neighborhood. + Also, the samples in X are not considered in the neighborhood of any + point. + The decision function on training data is available by considering the + opposite of the negative_outlier_factor_ attribute. + + Parameters + ---------- + X : array-like, shape (n_samples, n_features) + The query sample or samples to compute the Local Outlier Factor + w.r.t. the training samples. + + Returns + ------- + opposite_lof_scores : array, shape (n_samples,) + The opposite of the Local Outlier Factor of each input samples. + The lower, the more abnormal. + """ + check_is_fitted(self, ["threshold_", "negative_outlier_factor_", + "_distances_fit_X_"]) + + X = check_array(X, accept_sparse='csr') + + distances_X, neighbors_indices_X = ( + self.kneighbors(X, n_neighbors=self.n_neighbors_)) + X_lrd = self._local_reachability_density(distances_X, + neighbors_indices_X) + + lrd_ratios_array = (self._lrd[neighbors_indices_X] / + X_lrd[:, np.newaxis]) + + # as bigger is better: + return -np.mean(lrd_ratios_array, axis=1) + + def _local_reachability_density(self, distances_X, neighbors_indices): + """The local reachability density (LRD) + + The LRD of a sample is the inverse of the average reachability + distance of its k-nearest neighbors. + + Parameters + ---------- + distances_X : array, shape (n_query, self.n_neighbors) + Distances to the neighbors (in the training samples `self._fit_X`) + of each query point to compute the LRD. + + neighbors_indices : array, shape (n_query, self.n_neighbors) + Neighbors indices (of each query point) among training samples + self._fit_X. + + Returns + ------- + local_reachability_density : array, shape (n_samples,) + The local reachability density of each sample. + """ + dist_k = self._distances_fit_X_[neighbors_indices, + self.n_neighbors_ - 1] + reach_dist_array = np.maximum(distances_X, dist_k) + + # 1e-10 to avoid `nan' when when nb of duplicates > n_neighbors_: + return 1. / (np.mean(reach_dist_array, axis=1) + 1e-10) diff --git a/sklearn/neighbors/regression.py b/sklearn/neighbors/regression.py index b38de5acceaf3..d9ad9b5d25f2e 100644 --- a/sklearn/neighbors/regression.py +++ b/sklearn/neighbors/regression.py @@ -29,7 +29,7 @@ class KNeighborsRegressor(NeighborsBase, KNeighborsMixin, Parameters ---------- n_neighbors : int, optional (default = 5) - Number of neighbors to use by default for :meth:`k_neighbors` queries. + Number of neighbors to use by default for :meth:`kneighbors` queries. weights : str or callable weight function used in prediction. Possible values: diff --git a/sklearn/neighbors/tests/test_lof.py b/sklearn/neighbors/tests/test_lof.py new file mode 100644 index 0000000000000..1222cab53b628 --- /dev/null +++ b/sklearn/neighbors/tests/test_lof.py @@ -0,0 +1,120 @@ +# Authors: Nicolas Goix +# Alexandre Gramfort +# License: BSD 3 clause + +from math import sqrt +import numpy as np +from sklearn import neighbors + +from numpy.testing import assert_array_equal + +from sklearn import metrics +from sklearn.metrics import roc_auc_score + +from sklearn.utils import check_random_state +from sklearn.utils.testing import assert_greater +from sklearn.utils.testing import assert_array_almost_equal +from sklearn.utils.testing import assert_equal +from sklearn.utils.testing import assert_warns_message + +from sklearn.datasets import load_iris + + +# load the iris dataset +# and randomly permute it +rng = check_random_state(0) +iris = load_iris() +perm = rng.permutation(iris.target.size) +iris.data = iris.data[perm] +iris.target = iris.target[perm] + + +def test_lof(): + # Toy sample (the last two samples are outliers): + X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [5, 3], [-4, 2]] + + # Test LocalOutlierFactor: + clf = neighbors.LocalOutlierFactor(n_neighbors=5) + score = clf.fit(X).negative_outlier_factor_ + assert_array_equal(clf._fit_X, X) + + # Assert largest outlier score is smaller than smallest inlier score: + assert_greater(np.min(score[:-2]), np.max(score[-2:])) + + # Assert predict() works: + clf = neighbors.LocalOutlierFactor(contamination=0.25, + n_neighbors=5).fit(X) + assert_array_equal(clf._predict(), 6 * [1] + 2 * [-1]) + + +def test_lof_performance(): + # Generate train/test data + rng = check_random_state(2) + X = 0.3 * rng.randn(120, 2) + X_train = np.r_[X + 2, X - 2] + X_train = X[:100] + + # Generate some abnormal novel observations + X_outliers = rng.uniform(low=-4, high=4, size=(20, 2)) + X_test = np.r_[X[100:], X_outliers] + y_test = np.array([0] * 20 + [1] * 20) + + # fit the model + clf = neighbors.LocalOutlierFactor().fit(X_train) + + # predict scores (the lower, the more normal) + y_pred = -clf._decision_function(X_test) + + # check that roc_auc is good + assert_greater(roc_auc_score(y_test, y_pred), .99) + + +def test_lof_values(): + # toy samples: + X_train = [[1, 1], [1, 2], [2, 1]] + clf = neighbors.LocalOutlierFactor(n_neighbors=2).fit(X_train) + s_0 = 2. * sqrt(2.) / (1. + sqrt(2.)) + s_1 = (1. + sqrt(2)) * (1. / (4. * sqrt(2.)) + 1. / (2. + 2. * sqrt(2))) + # check predict() + assert_array_almost_equal(-clf.negative_outlier_factor_, [s_0, s_1, s_1]) + # check predict(one sample not in train) + assert_array_almost_equal(-clf._decision_function([[2., 2.]]), [s_0]) + # # check predict(one sample already in train) + assert_array_almost_equal(-clf._decision_function([[1., 1.]]), [s_1]) + + +def test_lof_precomputed(random_state=42): + """Tests LOF with a distance matrix.""" + # Note: smaller samples may result in spurious test success + rng = np.random.RandomState(random_state) + X = rng.random_sample((10, 4)) + Y = rng.random_sample((3, 4)) + DXX = metrics.pairwise_distances(X, metric='euclidean') + DYX = metrics.pairwise_distances(Y, X, metric='euclidean') + # As a feature matrix (n_samples by n_features) + lof_X = neighbors.LocalOutlierFactor(n_neighbors=3) + lof_X.fit(X) + pred_X_X = lof_X._predict() + pred_X_Y = lof_X._predict(Y) + + # As a dense distance matrix (n_samples by n_samples) + lof_D = neighbors.LocalOutlierFactor(n_neighbors=3, algorithm='brute', + metric='precomputed') + lof_D.fit(DXX) + pred_D_X = lof_D._predict() + pred_D_Y = lof_D._predict(DYX) + + assert_array_almost_equal(pred_X_X, pred_D_X) + assert_array_almost_equal(pred_X_Y, pred_D_Y) + + +def test_n_neighbors_attribute(): + X = iris.data + clf = neighbors.LocalOutlierFactor(n_neighbors=500).fit(X) + assert_equal(clf.n_neighbors_, X.shape[0] - 1) + + clf = neighbors.LocalOutlierFactor(n_neighbors=500) + assert_warns_message(UserWarning, + "n_neighbors will be set to (n_samples - 1)", + clf.fit, X) + assert_equal(clf.n_neighbors_, X.shape[0] - 1) diff --git a/sklearn/neighbors/unsupervised.py b/sklearn/neighbors/unsupervised.py index e20fcb9ab65ba..04d24d23b8df0 100644 --- a/sklearn/neighbors/unsupervised.py +++ b/sklearn/neighbors/unsupervised.py @@ -15,7 +15,7 @@ class NearestNeighbors(NeighborsBase, KNeighborsMixin, Parameters ---------- n_neighbors : int, optional (default = 5) - Number of neighbors to use by default for :meth:`k_neighbors` queries. + Number of neighbors to use by default for :meth:`kneighbors` queries. radius : float, optional (default = 1.0) Range of parameter space to use by default for :meth:`radius_neighbors` @@ -77,7 +77,7 @@ class NearestNeighbors(NeighborsBase, KNeighborsMixin, n_jobs : int, optional (default = 1) The number of parallel jobs to run for neighbors search. If ``-1``, then the number of jobs is set to the number of CPU cores. - Affects only :meth:`k_neighbors` and :meth:`kneighbors_graph` methods. + Affects only :meth:`kneighbors` and :meth:`kneighbors_graph` methods. Examples -------- diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py index 9f1d1320d196e..2b81446d8f871 100644 --- a/sklearn/neural_network/multilayer_perceptron.py +++ b/sklearn/neural_network/multilayer_perceptron.py @@ -562,7 +562,7 @@ def _fit_stochastic(self, X, y, activations, deltas, coef_grads, 'converged yet.' % (), ConvergenceWarning) except KeyboardInterrupt: - pass + warnings.warn("Training interrupted by user.") if early_stopping: # restore best weights diff --git a/sklearn/preprocessing/tests/test_imputation.py b/sklearn/preprocessing/tests/test_imputation.py index 028f5603d1bbd..1bfbcd3adbaee 100644 --- a/sklearn/preprocessing/tests/test_imputation.py +++ b/sklearn/preprocessing/tests/test_imputation.py @@ -6,14 +6,13 @@ from sklearn.utils.testing import assert_array_equal from sklearn.utils.testing import assert_raises from sklearn.utils.testing import assert_false -from sklearn.utils.testing import assert_true from sklearn.preprocessing.imputation import Imputer from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn import tree from sklearn.random_projection import sparse_random_matrix - + def _check_statistics(X, X_true, strategy, statistics, missing_values): @@ -92,16 +91,16 @@ def test_imputation_mean_median_only_zero(): # Test imputation using the mean and median strategies, when # missing_values == 0. X = np.array([ - [np.nan, 0, 0, 0, 5], - [np.nan, 1, 0, np.nan, 3], - [np.nan, 2, 0, 0, 0], - [np.nan, 6, 0, 5, 13], + [np.nan, 0, 0, 0, 5], + [np.nan, 1, 0, np.nan, 3], + [np.nan, 2, 0, 0, 0], + [np.nan, 6, 0, 5, 13], ]) X_imputed_mean = np.array([ - [3, 5], - [1, 3], - [2, 7], + [3, 5], + [1, 3], + [2, 7], [6, 13], ]) statistics_mean = [np.nan, 3, np.nan, np.nan, 7] @@ -144,7 +143,7 @@ def test_imputation_mean_median(): shape = (dim * dim, dim + dec) zeros = np.zeros(shape[0]) - values = np.arange(1, shape[0]+1) + values = np.arange(1, shape[0] + 1) values[4::2] = - values[4::2] tests = [("mean", "NaN", lambda z, v, p: safe_mean(np.hstack((z, v)))), @@ -236,17 +235,17 @@ def test_imputation_median_special_cases(): def test_imputation_most_frequent(): # Test imputation using the most-frequent strategy. X = np.array([ - [-1, -1, 0, 5], - [-1, 2, -1, 3], - [-1, 1, 3, -1], - [-1, 2, 3, 7], + [-1, -1, 0, 5], + [-1, 2, -1, 3], + [-1, 1, 3, -1], + [-1, 2, 3, 7], ]) X_true = np.array([ - [2, 0, 5], - [2, 3, 3], - [1, 3, 3], - [2, 3, 7], + [2, 0, 5], + [2, 3, 3], + [1, 3, 3], + [2, 3, 7], ]) # scipy.stats.mode, used in Imputer, doesn't return the first most @@ -315,7 +314,7 @@ def test_imputation_copy(): imputer = Imputer(missing_values=0, strategy="mean", copy=False) Xt = imputer.fit(X).transform(X) Xt[0, 0] = -1 - assert_true(np.all(X == Xt)) + assert_array_equal(X, Xt) # copy=False, sparse csr, axis=1 => no copy X = X_orig.copy() @@ -323,7 +322,7 @@ def test_imputation_copy(): copy=False, axis=1) Xt = imputer.fit(X).transform(X) Xt.data[0] = -1 - assert_true(np.all(X.data == Xt.data)) + assert_array_equal(X.data, Xt.data) # copy=False, sparse csc, axis=0 => no copy X = X_orig.copy().tocsc() @@ -331,7 +330,7 @@ def test_imputation_copy(): copy=False, axis=0) Xt = imputer.fit(X).transform(X) Xt.data[0] = -1 - assert_true(np.all(X.data == Xt.data)) + assert_array_equal(X.data, Xt.data) # copy=False, sparse csr, axis=0 => copy X = X_orig.copy() diff --git a/sklearn/qda.py b/sklearn/qda.py index 069afb629c7f4..604d6a919d261 100644 --- a/sklearn/qda.py +++ b/sklearn/qda.py @@ -1,6 +1,20 @@ import warnings +from .discriminant_analysis import QuadraticDiscriminantAnalysis as _QDA + warnings.warn("qda.QDA has been moved to " "discriminant_analysis.QuadraticDiscriminantAnalysis " "in 0.17 and will be removed in 0.19.", DeprecationWarning) -from .discriminant_analysis import QuadraticDiscriminantAnalysis as QDA + +class QDA(_QDA): + """ + Alias for + :class:`sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis`. + + .. deprecated:: 0.17 + This class will be removed in 0.19. + Use + :class:`sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis` + instead. + """ + pass diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 8a7d1babd24ed..a05429abc1d8d 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -28,12 +28,8 @@ from sklearn.linear_model.base import LinearClassifierMixin from sklearn.utils.estimator_checks import ( _yield_all_checks, - CROSS_DECOMPOSITION, check_parameters_default_constructible, - check_class_weight_balanced_linear_classifier, - check_transformer_n_iter, - check_non_transformer_estimators_n_iter, - check_get_params_invariance) + check_class_weight_balanced_linear_classifier) def test_all_estimator_no_base_class(): @@ -162,72 +158,3 @@ def test_all_tests_are_importable(): '{0} do not have `tests` subpackages. Perhaps they require ' '__init__.py or an add_subpackage directive in the parent ' 'setup.py'.format(missing_tests)) - - -def test_non_transformer_estimators_n_iter(): - # Test that all estimators of type which are non-transformer - # and which have an attribute of max_iter, return the attribute - # of n_iter atleast 1. - for est_type in ['regressor', 'classifier', 'cluster']: - regressors = all_estimators(type_filter=est_type) - for name, Estimator in regressors: - # LassoLars stops early for the default alpha=1.0 for - # the iris dataset. - if name == 'LassoLars': - estimator = Estimator(alpha=0.) - else: - with ignore_warnings(category=DeprecationWarning): - estimator = Estimator() - if hasattr(estimator, "max_iter"): - # These models are dependent on external solvers like - # libsvm and accessing the iter parameter is non-trivial. - if name in (['Ridge', 'SVR', 'NuSVR', 'NuSVC', - 'RidgeClassifier', 'SVC', 'RandomizedLasso', - 'LogisticRegressionCV']): - continue - - # Tested in test_transformer_n_iter below - elif (name in CROSS_DECOMPOSITION or - name in ['LinearSVC', 'LogisticRegression']): - continue - - else: - # Multitask models related to ENet cannot handle - # if y is mono-output. - yield (_named_check( - check_non_transformer_estimators_n_iter, name), - name, estimator, 'Multi' in name) - - -def test_transformer_n_iter(): - transformers = all_estimators(type_filter='transformer') - for name, Estimator in transformers: - with ignore_warnings(category=DeprecationWarning): - estimator = Estimator() - # Dependent on external solvers and hence accessing the iter - # param is non-trivial. - external_solver = ['Isomap', 'KernelPCA', 'LocallyLinearEmbedding', - 'RandomizedLasso', 'LogisticRegressionCV'] - - if hasattr(estimator, "max_iter") and name not in external_solver: - yield _named_check( - check_transformer_n_iter, name), name, estimator - - -def test_get_params_invariance(): - # Test for estimators that support get_params, that - # get_params(deep=False) is a subset of get_params(deep=True) - # Related to issue #4465 - - estimators = all_estimators(include_meta_estimators=False, - include_other=True) - for name, Estimator in estimators: - if hasattr(Estimator, 'get_params'): - # If class is deprecated, ignore deprecated warnings - if hasattr(Estimator.__init__, "deprecated_original"): - with ignore_warnings(): - yield _named_check( - check_get_params_invariance, name), name, Estimator - else: - yield _named_check( - check_get_params_invariance, name), name, Estimator diff --git a/sklearn/tests/test_discriminant_analysis.py b/sklearn/tests/test_discriminant_analysis.py index 24b7a1340526f..d1049fa6de31e 100644 --- a/sklearn/tests/test_discriminant_analysis.py +++ b/sklearn/tests/test_discriminant_analysis.py @@ -171,19 +171,17 @@ def test_lda_explained_variance_ratio(): clf_lda_eigen = LinearDiscriminantAnalysis(solver="eigen") clf_lda_eigen.fit(X, y) assert_almost_equal(clf_lda_eigen.explained_variance_ratio_.sum(), 1.0, 3) + assert_equal(clf_lda_eigen.explained_variance_ratio_.shape, (2,), + "Unexpected length for explained_variance_ratio_") clf_lda_svd = LinearDiscriminantAnalysis(solver="svd") clf_lda_svd.fit(X, y) assert_almost_equal(clf_lda_svd.explained_variance_ratio_.sum(), 1.0, 3) + assert_equal(clf_lda_svd.explained_variance_ratio_.shape, (2,), + "Unexpected length for explained_variance_ratio_") - tested_length = min(clf_lda_svd.explained_variance_ratio_.shape[0], - clf_lda_eigen.explained_variance_ratio_.shape[0]) - - # NOTE: clf_lda_eigen.explained_variance_ratio_ is not of n_components - # length. Make it the same length as clf_lda_svd.explained_variance_ratio_ - # before comparison. assert_array_almost_equal(clf_lda_svd.explained_variance_ratio_, - clf_lda_eigen.explained_variance_ratio_[:tested_length]) + clf_lda_eigen.explained_variance_ratio_) def test_lda_orthogonality(): @@ -331,7 +329,7 @@ def import_lda_module(): return sklearn.lda lda = assert_warns(DeprecationWarning, import_lda_module) - assert lda.LDA is LinearDiscriminantAnalysis + assert isinstance(lda.LDA(), LinearDiscriminantAnalysis) def import_qda_module(): import sklearn.qda @@ -341,7 +339,7 @@ def import_qda_module(): return sklearn.qda qda = assert_warns(DeprecationWarning, import_qda_module) - assert qda.QDA is QuadraticDiscriminantAnalysis + assert isinstance(qda.QDA(), QuadraticDiscriminantAnalysis) def test_covariance(): diff --git a/sklearn/tests/test_grid_search.py b/sklearn/tests/test_grid_search.py index 13b9086310595..e6c2e18538163 100644 --- a/sklearn/tests/test_grid_search.py +++ b/sklearn/tests/test_grid_search.py @@ -42,6 +42,7 @@ from sklearn.metrics import f1_score from sklearn.metrics import make_scorer from sklearn.metrics import roc_auc_score +from sklearn.linear_model import Ridge from sklearn.exceptions import ChangedBehaviorWarning from sklearn.exceptions import FitFailedWarning @@ -785,3 +786,20 @@ def test_parameters_sampler_replacement(): sampler = ParameterSampler(params_distribution, n_iter=7) samples = list(sampler) assert_equal(len(samples), 7) + + +def test_classes__property(): + # Test that classes_ property matches best_esimator_.classes_ + X = np.arange(100).reshape(10, 10) + y = np.array([0] * 5 + [1] * 5) + Cs = [.1, 1, 10] + + grid_search = GridSearchCV(LinearSVC(random_state=0), {'C': Cs}) + grid_search.fit(X, y) + assert_array_equal(grid_search.best_estimator_.classes_, + grid_search.classes_) + + # Test that regressors do not have a classes_ attribute + grid_search = GridSearchCV(Ridge(), {'alpha': [1.0, 2.0]}) + grid_search.fit(X, y) + assert_false(hasattr(grid_search, 'classes_')) diff --git a/sklearn/tree/_criterion.pxd b/sklearn/tree/_criterion.pxd index 889a623d732b3..cf6d32d1b7fe1 100644 --- a/sklearn/tree/_criterion.pxd +++ b/sklearn/tree/_criterion.pxd @@ -34,6 +34,7 @@ cdef class Criterion: cdef SIZE_t end cdef SIZE_t n_outputs # Number of outputs + cdef SIZE_t n_samples # Number of samples cdef SIZE_t n_node_samples # Number of samples in the node (end-start) cdef double weighted_n_samples # Weighted number of samples (in total) cdef double weighted_n_node_samples # Weighted number of samples in the node diff --git a/sklearn/tree/_criterion.pyx b/sklearn/tree/_criterion.pyx index 7e2b6a3a80e9e..e7ad82f6dcd49 100644 --- a/sklearn/tree/_criterion.pyx +++ b/sklearn/tree/_criterion.pyx @@ -235,6 +235,7 @@ cdef class ClassificationCriterion(Criterion): self.end = 0 self.n_outputs = n_outputs + self.n_samples = 0 self.n_node_samples = 0 self.weighted_n_node_samples = 0.0 self.weighted_n_left = 0.0 @@ -273,11 +274,10 @@ cdef class ClassificationCriterion(Criterion): def __dealloc__(self): """Destructor.""" - free(self.n_classes) def __reduce__(self): - return (ClassificationCriterion, + return (type(self), (self.n_outputs, sizet_ptr_to_ndarray(self.n_classes, self.n_outputs)), self.__getstate__()) @@ -710,6 +710,7 @@ cdef class RegressionCriterion(Criterion): self.end = 0 self.n_outputs = n_outputs + self.n_samples = n_samples self.n_node_samples = 0 self.weighted_n_node_samples = 0.0 self.weighted_n_left = 0.0 @@ -734,7 +735,7 @@ cdef class RegressionCriterion(Criterion): raise MemoryError() def __reduce__(self): - return (RegressionCriterion, (self.n_outputs,), self.__getstate__()) + return (type(self), (self.n_outputs, self.n_samples), self.__getstate__()) cdef void init(self, DOUBLE_t* y, SIZE_t y_stride, DOUBLE_t* sample_weight, double weighted_n_samples, SIZE_t* samples, SIZE_t start, @@ -881,6 +882,7 @@ cdef class MSE(RegressionCriterion): MSE = var_left + var_right """ + cdef double node_impurity(self) nogil: """Evaluate the impurity of the current node, i.e. the impurity of samples[start:end].""" @@ -1004,6 +1006,7 @@ cdef class MAE(RegressionCriterion): self.end = 0 self.n_outputs = n_outputs + self.n_samples = n_samples self.n_node_samples = 0 self.weighted_n_node_samples = 0.0 self.weighted_n_left = 0.0 diff --git a/sklearn/tree/_tree.pyx b/sklearn/tree/_tree.pyx index 4e8160f7dc780..f8632ab1640d8 100644 --- a/sklearn/tree/_tree.pyx +++ b/sklearn/tree/_tree.pyx @@ -547,8 +547,7 @@ cdef class Tree: # (i.e. through `_resize` or `__setstate__`) property n_classes: def __get__(self): - # it's small; copy for memory safety - return sizet_ptr_to_ndarray(self.n_classes, self.n_outputs).copy() + return sizet_ptr_to_ndarray(self.n_classes, self.n_outputs) property children_left: def __get__(self): @@ -839,8 +838,8 @@ cdef class Tree: # which features are nonzero in the present sample. cdef SIZE_t* feature_to_sample = NULL - safe_realloc(&X_sample, n_features * sizeof(DTYPE_t)) - safe_realloc(&feature_to_sample, n_features * sizeof(SIZE_t)) + safe_realloc(&X_sample, n_features) + safe_realloc(&feature_to_sample, n_features) with nogil: memset(feature_to_sample, -1, n_features * sizeof(SIZE_t)) @@ -985,8 +984,8 @@ cdef class Tree: # which features are nonzero in the present sample. cdef SIZE_t* feature_to_sample = NULL - safe_realloc(&X_sample, n_features * sizeof(DTYPE_t)) - safe_realloc(&feature_to_sample, n_features * sizeof(SIZE_t)) + safe_realloc(&X_sample, n_features) + safe_realloc(&feature_to_sample, n_features) with nogil: memset(feature_to_sample, -1, n_features * sizeof(SIZE_t)) diff --git a/sklearn/tree/_utils.pxd b/sklearn/tree/_utils.pxd index d11880908c318..fce3abcb734db 100644 --- a/sklearn/tree/_utils.pxd +++ b/sklearn/tree/_utils.pxd @@ -10,6 +10,7 @@ import numpy as np cimport numpy as np +from _tree cimport Node ctypedef np.npy_float32 DTYPE_t # Type of X ctypedef np.npy_float64 DOUBLE_t # Type of y, sample_weight @@ -36,6 +37,9 @@ ctypedef fused realloc_ptr: (unsigned char*) (WeightedPQueueRecord*) (DOUBLE_t*) + (DOUBLE_t**) + (Node*) + (Node**) cdef realloc_ptr safe_realloc(realloc_ptr* p, size_t nelems) except * diff --git a/sklearn/tree/_utils.pyx b/sklearn/tree/_utils.pyx index 9377cfa616e16..a4ccc71946bd1 100644 --- a/sklearn/tree/_utils.pyx +++ b/sklearn/tree/_utils.pyx @@ -62,10 +62,10 @@ cdef inline UINT32_t our_rand_r(UINT32_t* seed) nogil: cdef inline np.ndarray sizet_ptr_to_ndarray(SIZE_t* data, SIZE_t size): - """Encapsulate data into a 1D numpy array of intp's.""" + """Return copied data as 1D numpy array of intp's.""" cdef np.npy_intp shape[1] shape[0] = size - return np.PyArray_SimpleNewFromData(1, shape, np.NPY_INTP, data) + return np.PyArray_SimpleNewFromData(1, shape, np.NPY_INTP, data).copy() cdef inline SIZE_t rand_int(SIZE_t low, SIZE_t high, diff --git a/sklearn/tree/tests/test_tree.py b/sklearn/tree/tests/test_tree.py index 7f7d9a124d513..c3e8e795b32f0 100644 --- a/sklearn/tree/tests/test_tree.py +++ b/sklearn/tree/tests/test_tree.py @@ -1,6 +1,7 @@ """ Testing for the tree module (sklearn.tree). """ +import copy import pickle from functools import partial from itertools import product @@ -42,12 +43,14 @@ from sklearn import tree from sklearn.tree._tree import TREE_LEAF +from sklearn.tree.tree import CRITERIA_CLF +from sklearn.tree.tree import CRITERIA_REG from sklearn import datasets from sklearn.utils import compute_sample_weight CLF_CRITERIONS = ("gini", "entropy") -REG_CRITERIONS = ("mse", "mae") +REG_CRITERIONS = ("mse", "mae", "friedman_mse") CLF_TREES = { "DecisionTreeClassifier": DecisionTreeClassifier, @@ -1597,6 +1600,7 @@ def test_no_sparse_y_support(): for name in ALL_TREES: yield (check_no_sparse_y_support, name) + def test_mae(): # check MAE criterion produces correct results # on small toy dataset @@ -1609,3 +1613,30 @@ def test_mae(): dt_mae.fit([[3],[5],[3],[8],[5]],[6,7,3,4,3], [0.6,0.3,0.1,1.0,0.3]) assert_array_equal(dt_mae.tree_.impurity, [7.0/2.3, 3.0/0.7, 4.0/1.6]) assert_array_equal(dt_mae.tree_.value.flat, [4.0, 6.0, 4.0]) + + +def test_criterion_copy(): + # Let's check whether copy of our criterion has the same type + # and properties as original + n_outputs = 3 + n_classes = np.arange(3, dtype=np.intp) + n_samples = 100 + + def _pickle_copy(obj): + return pickle.loads(pickle.dumps(obj)) + for copy_func in [copy.copy, copy.deepcopy, _pickle_copy]: + for _, typename in CRITERIA_CLF.items(): + criteria = typename(n_outputs, n_classes) + result = copy_func(criteria).__reduce__() + typename_, (n_outputs_, n_classes_), _ = result + assert_equal(typename, typename_) + assert_equal(n_outputs, n_outputs_) + assert_array_equal(n_classes, n_classes_) + + for _, typename in CRITERIA_REG.items(): + criteria = typename(n_outputs, n_samples) + result = copy_func(criteria).__reduce__() + typename_, (n_outputs_, n_samples_), _ = result + assert_equal(typename, typename_) + assert_equal(n_outputs, n_outputs_) + assert_equal(n_samples, n_samples_) diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index 04a234a16b423..6c58cea6032c9 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -132,6 +132,8 @@ def _yield_classifier_checks(name, Classifier): if 'class_weight' in Classifier().get_params().keys(): yield check_class_weight_classifiers + yield check_non_transformer_estimators_n_iter + @ignore_warnings(category=DeprecationWarning) def check_supervised_y_no_nan(name, Estimator): @@ -172,6 +174,7 @@ def _yield_regressor_checks(name, Regressor): if name != "GaussianProcessRegressor": # Test if NotFittedError is raised yield check_estimators_unfitted + yield check_non_transformer_estimators_n_iter def _yield_transformer_checks(name, Transformer): @@ -186,6 +189,13 @@ def _yield_transformer_checks(name, Transformer): # basic tests yield check_transformer_general yield check_transformers_unfitted + # Dependent on external solvers and hence accessing the iter + # param is non-trivial. + external_solver = ['Isomap', 'KernelPCA', 'LocallyLinearEmbedding', + 'RandomizedLasso', 'LogisticRegressionCV'] + if name not in external_solver: + yield check_transformer_n_iter + def _yield_clustering_checks(name, Clusterer): @@ -195,6 +205,7 @@ def _yield_clustering_checks(name, Clusterer): # let's not test that here. yield check_clustering yield check_estimators_partial_fit_n_features + yield check_non_transformer_estimators_n_iter def _yield_all_checks(name, Estimator): @@ -218,6 +229,7 @@ def _yield_all_checks(name, Estimator): yield check_fit2d_1feature yield check_fit1d_1feature yield check_fit1d_1sample + yield check_get_params_invariance def check_estimator(Estimator): @@ -1477,51 +1489,73 @@ def multioutput_estimator_convert_y_2d(name, y): @ignore_warnings(category=DeprecationWarning) -def check_non_transformer_estimators_n_iter(name, estimator, - multi_output=False): - # Check if all iterative solvers, run for more than one iteration - - iris = load_iris() - X, y_ = iris.data, iris.target - - if multi_output: - y_ = np.reshape(y_, (-1, 1)) +def check_non_transformer_estimators_n_iter(name, Estimator): + # Test that estimators that are not transformers with a parameter + # max_iter, return the attribute of n_iter_ at least 1. + + # These models are dependent on external solvers like + # libsvm and accessing the iter parameter is non-trivial. + not_run_check_n_iter = ['Ridge', 'SVR', 'NuSVR', 'NuSVC', + 'RidgeClassifier', 'SVC', 'RandomizedLasso', + 'LogisticRegressionCV', 'LinearSVC', + 'LogisticRegression'] + + # Tested in test_transformer_n_iter + not_run_check_n_iter += CROSS_DECOMPOSITION + if name in not_run_check_n_iter: + return - set_random_state(estimator, 0) - if name == 'AffinityPropagation': - estimator.fit(X) + # LassoLars stops early for the default alpha=1.0 the iris dataset. + if name == 'LassoLars': + estimator = Estimator(alpha=0.) else: - estimator.fit(X, y_) + estimator = Estimator() + if hasattr(estimator, 'max_iter'): + iris = load_iris() + X, y_ = iris.data, iris.target + y_ = multioutput_estimator_convert_y_2d(name, y_) + + set_random_state(estimator, 0) + if name == 'AffinityPropagation': + estimator.fit(X) + else: + estimator.fit(X, y_) - # HuberRegressor depends on scipy.optimize.fmin_l_bfgs_b - # which doesn't return a n_iter for old versions of SciPy. - if not (name == 'HuberRegressor' and estimator.n_iter_ is None): - assert_greater_equal(estimator.n_iter_, 1) + # HuberRegressor depends on scipy.optimize.fmin_l_bfgs_b + # which doesn't return a n_iter for old versions of SciPy. + if not (name == 'HuberRegressor' and estimator.n_iter_ is None): + assert_greater_equal(estimator.n_iter_, 1) @ignore_warnings(category=DeprecationWarning) -def check_transformer_n_iter(name, estimator): - if name in CROSS_DECOMPOSITION: - # Check using default data - X = [[0., 0., 1.], [1., 0., 0.], [2., 2., 2.], [2., 5., 4.]] - y_ = [[0.1, -0.2], [0.9, 1.1], [0.1, -0.5], [0.3, -0.2]] +def check_transformer_n_iter(name, Estimator): + # Test that transformers with a parameter max_iter, return the + # attribute of n_iter_ at least 1. + estimator = Estimator() + if hasattr(estimator, "max_iter"): + if name in CROSS_DECOMPOSITION: + # Check using default data + X = [[0., 0., 1.], [1., 0., 0.], [2., 2., 2.], [2., 5., 4.]] + y_ = [[0.1, -0.2], [0.9, 1.1], [0.1, -0.5], [0.3, -0.2]] - else: - X, y_ = make_blobs(n_samples=30, centers=[[0, 0, 0], [1, 1, 1]], - random_state=0, n_features=2, cluster_std=0.1) - X -= X.min() - 0.1 - set_random_state(estimator, 0) - estimator.fit(X, y_) + else: + X, y_ = make_blobs(n_samples=30, centers=[[0, 0, 0], [1, 1, 1]], + random_state=0, n_features=2, cluster_std=0.1) + X -= X.min() - 0.1 + set_random_state(estimator, 0) + estimator.fit(X, y_) - # These return a n_iter per component. - if name in CROSS_DECOMPOSITION: - for iter_ in estimator.n_iter_: - assert_greater_equal(iter_, 1) - else: - assert_greater_equal(estimator.n_iter_, 1) + # These return a n_iter per component. + if name in CROSS_DECOMPOSITION: + for iter_ in estimator.n_iter_: + assert_greater_equal(iter_, 1) + else: + assert_greater_equal(estimator.n_iter_, 1) +@ignore_warnings(category=DeprecationWarning) def check_get_params_invariance(name, estimator): + # Checks if get_params(deep=False) is a subset of get_params(deep=True) class T(BaseEstimator): """Mock classifier """ diff --git a/sklearn/utils/extmath.py b/sklearn/utils/extmath.py index 1857a27adfadc..741601531d975 100644 --- a/sklearn/utils/extmath.py +++ b/sklearn/utils/extmath.py @@ -25,7 +25,7 @@ from ..externals.six.moves import xrange from .sparsefuncs_fast import csr_row_norms from .validation import check_array -from ..exceptions import NonBLASDotWarning +from ..exceptions import ConvergenceWarning, NonBLASDotWarning def norm(x): @@ -844,21 +844,30 @@ def _deterministic_vector_sign_flip(u): return u -def stable_cumsum(arr, rtol=1e-05, atol=1e-08): +def stable_cumsum(arr, axis=None, rtol=1e-05, atol=1e-08): """Use high precision for cumsum and check that final value matches sum Parameters ---------- arr : array-like To be cumulatively summed as flat + axis : int, optional + Axis along which the cumulative sum is computed. + The default (None) is to compute the cumsum over the flattened array. rtol : float Relative tolerance, see ``np.allclose`` atol : float Absolute tolerance, see ``np.allclose`` """ - out = np.cumsum(arr, dtype=np.float64) - expected = np.sum(arr, dtype=np.float64) - if not np.allclose(out[-1], expected, rtol=rtol, atol=atol): - raise RuntimeError('cumsum was found to be unstable: ' - 'its last element does not correspond to sum') + # sum is as unstable as cumsum for numpy < 1.9 + if np_version < (1, 9): + return np.cumsum(arr, axis=axis, dtype=np.float64) + + out = np.cumsum(arr, axis=axis, dtype=np.float64) + expected = np.sum(arr, axis=axis, dtype=np.float64) + if not np.all(np.isclose(out.take(-1, axis=axis), expected, rtol=rtol, + atol=atol, equal_nan=True)): + warnings.warn('cumsum was found to be unstable: ' + 'its last element does not correspond to sum', + ConvergenceWarning) return out diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index 682ab7733c77a..c7bc8f3078d6b 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -419,3 +419,33 @@ def __getstate__(self): self._fill_value) else: from numpy.ma import MaskedArray # noqa + +if 'axis' not in signature(np.linalg.norm).parameters: + + def norm(X, ord=None, axis=None): + """ + Handles the axis parameter for the norm function + in old versions of numpy (useless for numpy >= 1.8). + """ + + if axis is None or X.ndim == 1: + result = np.linalg.norm(X, ord=ord) + return result + + if axis not in (0, 1): + raise NotImplementedError(""" + The fix that adds axis parameter to the old numpy + norm only works for 1D or 2D arrays. + """) + + if axis == 0: + X = X.T + + result = np.zeros(X.shape[0]) + for i in range(len(result)): + result[i] = np.linalg.norm(X[i], ord=ord) + + return result + +else: + norm = np.linalg.norm diff --git a/sklearn/utils/stats.py b/sklearn/utils/stats.py index 463146d038c6b..265d193e6bf01 100644 --- a/sklearn/utils/stats.py +++ b/sklearn/utils/stats.py @@ -1,6 +1,7 @@ import numpy as np from scipy.stats import rankdata as _sp_rankdata from .fixes import bincount +from ..utils.extmath import stable_cumsum # To remove when we support scipy 0.13 @@ -53,7 +54,7 @@ def _weighted_percentile(array, sample_weight, percentile=50): sorted_idx = np.argsort(array) # Find index of median prediction for each sample - weight_cdf = sample_weight[sorted_idx].cumsum() + weight_cdf = stable_cumsum(sample_weight[sorted_idx]) percentile_idx = np.searchsorted( weight_cdf, (percentile / 100.) * weight_cdf[-1]) return array[sorted_idx[percentile_idx]] diff --git a/sklearn/utils/tests/test_extmath.py b/sklearn/utils/tests/test_extmath.py index 55f96cdf1574c..f88d3ac3e10ea 100644 --- a/sklearn/utils/tests/test_extmath.py +++ b/sklearn/utils/tests/test_extmath.py @@ -17,7 +17,7 @@ from sklearn.utils.testing import assert_false from sklearn.utils.testing import assert_greater from sklearn.utils.testing import assert_raises -from sklearn.utils.testing import assert_raise_message +from sklearn.utils.testing import assert_warns from sklearn.utils.testing import skip_if_32bit from sklearn.utils.testing import SkipTest from sklearn.utils.fixes import np_version @@ -36,6 +36,7 @@ from sklearn.utils.extmath import _deterministic_vector_sign_flip from sklearn.utils.extmath import softmax from sklearn.utils.extmath import stable_cumsum +from sklearn.exceptions import ConvergenceWarning from sklearn.datasets.samples_generator import make_low_rank_matrix @@ -63,8 +64,8 @@ def test_uniform_weights(): mode, score = stats.mode(x, axis) mode2, score2 = weighted_mode(x, weights, axis) - assert_true(np.all(mode == mode2)) - assert_true(np.all(score == score2)) + assert_array_equal(mode, mode2) + assert_array_equal(score, score2) def test_random_weights(): @@ -654,7 +655,10 @@ def test_stable_cumsum(): raise SkipTest("Sum is as unstable as cumsum for numpy < 1.9") assert_array_equal(stable_cumsum([1, 2, 3]), np.cumsum([1, 2, 3])) r = np.random.RandomState(0).rand(100000) - assert_raise_message(RuntimeError, - 'cumsum was found to be unstable: its last element ' - 'does not correspond to sum', - stable_cumsum, r, rtol=0, atol=0) + assert_warns(ConvergenceWarning, stable_cumsum, r, rtol=0, atol=0) + + # test axis parameter + A = np.random.RandomState(36).randint(1000, size=(5, 5, 5)) + assert_array_equal(stable_cumsum(A, axis=0), np.cumsum(A, axis=0)) + assert_array_equal(stable_cumsum(A, axis=1), np.cumsum(A, axis=1)) + assert_array_equal(stable_cumsum(A, axis=2), np.cumsum(A, axis=2)) diff --git a/sklearn/utils/tests/test_fixes.py b/sklearn/utils/tests/test_fixes.py index ef1110bfc4eed..b7e2dd7180776 100644 --- a/sklearn/utils/tests/test_fixes.py +++ b/sklearn/utils/tests/test_fixes.py @@ -5,6 +5,7 @@ import pickle import numpy as np +import math from sklearn.utils.testing import assert_equal from sklearn.utils.testing import assert_false @@ -16,6 +17,7 @@ from sklearn.utils.fixes import divide, expit from sklearn.utils.fixes import astype from sklearn.utils.fixes import MaskedArray +from sklearn.utils.fixes import norm def test_expit(): @@ -66,3 +68,26 @@ def test_masked_array_obj_dtype_pickleable(): marr_pickled = pickle.loads(pickle.dumps(marr)) assert_array_equal(marr.data, marr_pickled.data) assert_array_equal(marr.mask, marr_pickled.mask) + + +def test_norm(): + X = np.array([[-2, 4, 5], + [1, 3, -4], + [0, 0, 8], + [0, 0, 0]]).astype(float) + + # Test various axis and order + assert_equal(math.sqrt(135), norm(X)) + assert_array_equal( + np.array([math.sqrt(5), math.sqrt(25), math.sqrt(105)]), + norm(X, axis=0) + ) + assert_array_equal(np.array([3, 7, 17]), norm(X, axis=0, ord=1)) + assert_array_equal(np.array([2, 4, 8]), norm(X, axis=0, ord=np.inf)) + assert_array_equal(np.array([0, 0, 0]), norm(X, axis=0, ord=-np.inf)) + assert_array_equal(np.array([11, 8, 8, 0]), norm(X, axis=1, ord=1)) + + # Test shapes + assert_equal((), norm(X).shape) + assert_equal((3,), norm(X, axis=0).shape) + assert_equal((4,), norm(X, axis=1).shape)