diff --git a/.gitignore b/.gitignore index 92a841500..4555e5cb6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*~ doc/generated examples/.ipynb_checkpoints # Byte-compiled / optimized / DLL files diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..2a215a985 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,178 @@ +How to contribute +----------------- + +The preferred workflow for contributing to the OpenML python connector is to +fork the [main repository](https://github.com/openml/openml-python) on +GitHub, clone, check out the branch `develop`, and develop on a new branch +branch. Steps: + +1. Fork the [project repository](https://github.com/openml/openml-python) + by clicking on the 'Fork' button near the top right of the page. This creates + a copy of the code under your GitHub user account. For more details on + how to fork a repository see [this guide](https://help.github.com/articles/fork-a-repo/). + +2. Clone your fork of the openml-python repo from your GitHub account to your +local disk: + + ```bash + $ git clone git@github.com:YourLogin/openml-python.git + $ cd openml-python + ``` + +3. Swith to the ``develop`` branch: + + ```bash + $ git checkout develop + ``` + +3. Create a ``feature`` branch to hold your development changes: + + ```bash + $ git checkout -b feature/my-feature + ``` + + Always use a ``feature`` branch. It's good practice to never work on the ``master`` or ``develop`` branch! To make the nature of your pull request easily visible, please perpend the name of the branch with the type of changes you want to merge, such as ``feature`` if it contains a new feature, ``fix`` for a bugfix, ``doc`` for documentation and ``maint`` for other maintenance on the package. + +4. Develop the feature on your feature branch. Add changed files using ``git add`` and then ``git commit`` files: + + ```bash + $ git add modified_files + $ git commit + ``` + + to record your changes in Git, then push the changes to your GitHub account with: + + ```bash + $ git push -u origin my-feature + ``` + +5. Follow [these instructions](https://help.github.com/articles/creating-a-pull-request-from-a-fork) +to create a pull request from your fork. This will send an email to the committers. + +(If any of the above seems like magic to you, please look up the +[Git documentation](https://git-scm.com/documentation) on the web, or ask a friend or another contributor for help.) + +Pull Request Checklist +---------------------- + +We recommended that your contribution complies with the +following rules before you submit a pull request: + +- Follow the + [pep8 style guilde](https://www.python.org/dev/peps/pep-0008/). + +- If your pull request addresses an issue, please use the pull request title + to describe the issue and mention the issue number in the pull request description. This will make sure a link back to the original issue is + created. + +- An incomplete contribution -- where you expect to do more work before + receiving a full review -- should be prefixed `[WIP]` (to indicate a work + in progress) and changed to `[MRG]` when it matures. WIPs may be useful + to: indicate you are working on something to avoid duplicated work, + request broad review of functionality or API, or seek collaborators. + WIPs often benefit from the inclusion of a + [task list](https://github.com/blog/1375-task-lists-in-gfm-issues-pulls-comments) + in the PR description. + +- All tests pass when running `nosetests`. On + Unix-like systems, check with (from the toplevel source folder): + + ```bash + $ nosetests + ``` + + For Windows systems, execute the command from an Anaconda Prompt or add `nosetests` to PATH before executing the command. + +- Documentation and high-coverage tests are necessary for enhancements to be + accepted. Bug-fixes or new features should be provided with + [non-regression tests](https://en.wikipedia.org/wiki/Non-regression_testing). + These tests verify the correct behavior of the fix or feature. In this + manner, further modifications on the code base are granted to be consistent + with the desired behavior. + For the Bug-fixes case, at the time of the PR, this tests should fail for + the code base in develop and pass for the PR code. + + +You can also check for common programming errors with the following +tools: + +- Code with good unittest **coverage** (at least 80%), check with: + + ```bash + $ pip install nose coverage + $ nosetests --with-coverage path/to/tests_for_package + ``` + +- No pyflakes warnings, check with: + + ```bash + $ pip install pyflakes + $ pyflakes path/to/module.py + ``` + +- No PEP8 warnings, check with: + + ```bash + $ pip install pep8 + $ pep8 path/to/module.py + ``` + +Filing bugs +----------- +We use GitHub issues to track all bugs and feature requests; feel free to +open an issue if you have found a bug or wish to see a feature implemented. + +It is recommended to check that your issue complies with the +following rules before submitting: + +- Verify that your issue is not being currently addressed by other + [issues](https://github.com/openml/openml-python/issues) + or [pull requests](https://github.com/openml/openml-python/pulls). + +- Please ensure all code snippets and error messages are formatted in + appropriate code blocks. + See [Creating and highlighting code blocks](https://help.github.com/articles/creating-and-highlighting-code-blocks). + +- Please include your operating system type and version number, as well + as your Python, openml, scikit-learn, numpy, and scipy versions. This information + can be found by running the following code snippet: + + ```python + import platform; print(platform.platform()) + import sys; print("Python", sys.version) + import numpy; print("NumPy", numpy.__version__) + import scipy; print("SciPy", scipy.__version__) + import sklearn; print("Scikit-Learn", sklearn.__version__) + import openml; print("OpenML", openml.__version__) + ``` + +New contributor tips +-------------------- + +A great way to start contributing to scikit-learn is to pick an item +from the list of [Easy issues](https://github.com/openml/openml-python/issues?q=label%3Aeasy) +in the issue tracker. Resolving these issues allow you to start +contributing to the project without much prior knowledge. Your +assistance in this area will be greatly appreciated by the more +experienced developers as it helps free up their time to concentrate on +other issues. + +Documentation +------------- + +We are glad to accept any sort of documentation: function docstrings, +reStructuredText documents (like this one), tutorials, etc. +reStructuredText documents live in the source code repository under the +doc/ directory. + +You can edit the documentation using any text editor and then generate +the HTML output by typing ``make html`` from the doc/ directory. +The resulting HTML files will be placed in ``build/html/`` and are viewable in +a web browser. See the ``README`` file in the ``doc/`` directory for more +information. + +For building the documentation, you will need +[sphinx](http://sphinx.pocoo.org/), +[matplotlib](http://matplotlib.org/), and +[pillow](http://pillow.readthedocs.io/en/latest/). +[sphinx-bootstrap-theme](https://ryan-roemer.github.io/sphinx-bootstrap-theme/) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..bcd5e0c1e --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,33 @@ +#### Description + + +#### Steps/Code to Reproduce + + +#### Expected Results + + +#### Actual Results + + +#### Versions + + + + \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..146b8cc36 --- /dev/null +++ b/LICENSE @@ -0,0 +1,68 @@ +BSD 3-Clause License + +Copyright (c) 2014-2018, Matthias Feurer, Jan van Rijn, Andreas Müller, +Joaquin Vanschoren and others. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License of the files CONTRIBUTING.md, ISSUE_TEMPLATE.md and +PULL_REQUEST_TEMPLATE.md: + +Those files are modifications of the respecting templates in scikit-learn and +they are licensed under a New BSD license: + +New BSD License + +Copyright (c) 2007–2018 The scikit-learn developers. +All rights reserved. + + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of the Scikit-learn Developers nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..c73beebea --- /dev/null +++ b/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ + +#### Reference Issue + + + +#### What does this PR implement/fix? Explain your changes. + + +#### How should this PR be tested? + + +#### Any other comments? + diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..e89e6fc7d --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,52 @@ + +environment: + global: + CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\scikit-learn-contrib\\run_with_env.cmd" + + matrix: + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + PYTHON_ARCH: "64" + MINICONDA: "C:\\Miniconda35-x64" + + - PYTHON: "C:\\Python35" + PYTHON_VERSION: "3.5" + PYTHON_ARCH: "32" + MINICONDA: "C:\\Miniconda35" + +matrix: + fast_finish: true + + +install: + # Miniconda is pre-installed in the worker build + - "SET PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" + - "python -m pip install -U pip" + + # Check that we have the expected version and architecture for Python + - "python --version" + - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" + - "pip --version" + + # Remove cygwin because it clashes with conda + # see http://help.appveyor.com/discussions/problems/3712-git-remote-https-seems-to-be-broken + - rmdir C:\\cygwin /s /q + + # Update previous packages and install the build and runtime dependencies of the project. + # XXX: setuptools>23 is currently broken on Win+py3 with numpy + # (https://github.com/pypa/setuptools/issues/728) + - conda update --all --yes setuptools=23 + + # Install the build and runtime dependencies of the project. + - "cd C:\\projects\\openml-python" + - conda install --quiet --yes mock numpy scipy nose requests scikit-learn nbformat python-dateutil nbconvert + - pip install liac-arff xmltodict oslo.concurrency + - "%CMD_IN_ENV% python setup.py install" + + +# Not a .NET project, we build scikit-learn in the install step instead +build: false + +test_script: + - "cd C:\\projects\\openml-python" + - "%CMD_IN_ENV% python setup.py test" diff --git a/appveyor/run_with_env.cmd b/appveyor/run_with_env.cmd new file mode 100644 index 000000000..5da547c49 --- /dev/null +++ b/appveyor/run_with_env.cmd @@ -0,0 +1,88 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific +:: environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +:: +:: Notes about batch files for Python people: +:: +:: Quotes in values are literally part of the values: +:: SET FOO="bar" +:: FOO is now five characters long: " b a r " +:: If you don't want quotes, don't include them on the right-hand side. +:: +:: The CALL lines at the end of this file look redundant, but if you move them +:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y +:: case, I don't know why. +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows +SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf + +:: Extract the major and minor versions, and allow for the minor version to be +:: more than 9. This requires the version number to have two dots in it. +SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% +IF "%PYTHON_VERSION:~3,1%" == "." ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% +) ELSE ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% +) + +:: Based on the Python version, determine what SDK version to use, and whether +:: to set the SDK for 64-bit. +IF %MAJOR_PYTHON_VERSION% == 2 ( + SET WINDOWS_SDK_VERSION="v7.0" + SET SET_SDK_64=Y +) ELSE ( + IF %MAJOR_PYTHON_VERSION% == 3 ( + SET WINDOWS_SDK_VERSION="v7.1" + IF %MINOR_PYTHON_VERSION% LEQ 4 ( + SET SET_SDK_64=Y + ) ELSE ( + SET SET_SDK_64=N + IF EXIST "%WIN_WDK%" ( + :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ + REN "%WIN_WDK%" 0wdf + ) + ) + ) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 + ) +) + +IF %PYTHON_ARCH% == 64 ( + IF %SET_SDK_64% == Y ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) ELSE ( + ECHO Using default MSVC build environment for 64 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) diff --git a/circle.yml b/circle.yml index cc61c92a1..ce5279bf1 100644 --- a/circle.yml +++ b/circle.yml @@ -25,14 +25,11 @@ dependencies: - pip install --upgrade pip - pip install --upgrade numpy - pip install --upgrade scipy - - pip install git+https://github.com/mfeurer/liac-arff.git # install documentation building dependencies - pip install --upgrade matplotlib setuptools nose coverage sphinx pillow sphinx-gallery sphinx_bootstrap_theme cython numpydoc scikit-learn nbformat nbconvert # Installing required packages for `make -C doc check command` to work. - sudo -E apt-get -yq update - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra - # finally install the requirements of the package to allow autodoc - - pip install -r requirements.txt # The --user is needed to let sphinx see the source and the binaries # The pipefail is requested to propagate exit code @@ -58,8 +55,3 @@ general: artifacts: - "doc/_build/html" - "~/log.txt" - # Restric the build to the branch master only - branches: - only: - - master - - develop diff --git a/doc/api.rst b/doc/api.rst index 79d59577c..4939cd99e 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -18,6 +18,7 @@ Top-level Classes OpenMLTask OpenMLSplit OpenMLFlow + OpenMLEvaluation :mod:`openml.datasets`: Dataset Functions @@ -33,6 +34,30 @@ Top-level Classes get_datasets list_datasets +:mod:`openml.evaluations`: Evaluation Functions +----------------------------------------------- +.. currentmodule:: openml.evaluations + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + list_evaluations + +:mod:`openml.flows`: Flow Functions +----------------------------------- +.. currentmodule:: openml.flows + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + flow_exists + flow_to_sklearn + get_flow + list_flows + sklearn_to_flow + :mod:`openml.runs`: Run Functions ---------------------------------- .. currentmodule:: openml.runs @@ -41,14 +66,37 @@ Top-level Classes :toctree: generated/ :template: function.rst - run_task - get_run - list_runs - list_runs_by_flow - list_runs_by_tag - list_runs_by_task - list_runs_by_uploader - list_runs_by_filters + get_run + get_runs + get_run_trace + initialize_model_from_run + initialize_model_from_trace + list_runs + run_model_on_task + run_flow_on_task + +:mod:`openml.setups`: Setup Functions +------------------------------------- +.. currentmodule:: openml.setups + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + get_setup + initialize_model + list_setups + setup_exists + +:mod:`openml.study`: Study Functions +------------------------------------ +.. currentmodule:: openml.study + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + get_study :mod:`openml.tasks`: Task Functions ----------------------------------- @@ -59,13 +107,8 @@ Top-level Classes :template: function.rst get_task + get_tasks list_tasks -:mod:`openml.flows`: Flow Functions ------------------------------------ -.. currentmodule:: openml.flow -.. autosummary:: - :toctree: generated/ - :template: function.rst diff --git a/doc/index.rst b/doc/index.rst index ef2f0cd50..3990fc09a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -3,11 +3,39 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to OpenML's documentation! -================================== - -The OpenML module is still under development. You can watch its progress -:ref:`here `. +====== +OpenML +====== + +Welcome to the documentation of the OpenML Python API, a connector to the +collaborative machine learning platform `OpenML.org `_. +The OpenML Python package allows to use datasets and tasks from OpenML together +with scikit-learn and share the results online. + +------- +Example +------- + +.. code:: python + + # Define a scikit-learn pipeline + clf = sklearn.pipeline.Pipeline( + steps=[ + ('imputer', sklearn.preprocessing.Imputer()), + ('estimator', sklearn.tree.DecisionTreeClassifier()) + ] + ) + # Download the OpenML task for the german credit card dataset with 10-fold + # cross-validation. + task = openml.tasks.get_task(31) + # Set the OpenML API Key which is required to upload the runs. + # You can get your own API by signing up to OpenML.org. + openml.config.apikey = 'ABC' + # Run the scikit-learn model on the task (requires an API key). + run = openml.runs.run_model_on_task(task, clf) + # Publish the experiment on OpenML (optional, requires an API key). + run.publish() + print('URL for run: %s/run/%d' % (openml.config.server, run.run_id)) ------------ diff --git a/doc/usage.rst b/doc/usage.rst index 98453f4d0..a4bf8ee0b 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -31,7 +31,6 @@ programmatically after loading the package: .. code:: python >>> import openml - >>> apikey = 'Your API key' >>> openml.config.apikey = apikey @@ -56,7 +55,7 @@ API: .. code:: python >>> import os - >>> openml.config.set_cache_directory(os.path.expanduser('~/.openml/cache')) + >>> openml.config.cache_directory = os.path.expanduser('~/.openml/cache') Config file: @@ -64,191 +63,44 @@ Config file: cachedir = '~/.openml/cache' -~~~~~~~~~~~~~~~~~~~~~ -Working with datasets -~~~~~~~~~~~~~~~~~~~~~ - -# TODO mention third, searching for tags - -Datasets are a key concept in OpenML (see `OpenML documentation `_). -Datasets are identified by IDs and can be accessed in two different ways: - -1. In a list providing basic information on all datasets available on OpenML. - This function will not download the actual dataset, but will instead download - meta data which can be used to filter the datasets and retrieve a set of IDs. -2. A single dataset by its ID. A single dataset contains all meta information and the actual - data in form of an .arff file. The .arff file will be converted into a numpy - array by the OpenML Python API. - -Listing datasets -~~~~~~~~~~~~~~~~ - -A common task when using OpenML is to find a set of datasets which fulfill -several criteria. They should for example have between 1,000 and 10,000 -data points and at least five features. - -.. code:: python - - >>> datasets = openml.datasets.list_datasets() - -:meth:`openml.datasets.list_datasets` returns a dictionary of dictionaries, we -will convert it into a -`pandas dataframe `_ -to have better visualization and easier access: - -.. code:: python - - >>> import pandas as pd - >>> datasets = pd.DataFrame.from_dict(datasets, orient='index') - -We have access to the following properties of the datasets: - - >>> print(datasets.columns) - Index(['did', 'name', 'format', 'status', 'MajorityClassSize', - 'MaxNominalAttDistinctValues', 'MinorityClassSize', 'NumberOfClasses', - 'NumberOfFeatures', 'NumberOfInstances', - 'NumberOfInstancesWithMissingValues', 'NumberOfMissingValues', - 'NumberOfNumericFeatures', 'NumberOfSymbolicFeatures'], - dtype='object') - -and can see the first data point: - - >>> print(datasets.iloc[0]) - did 2 - name anneal - format ARFF - status active - MajorityClassSize 684 - MaxNominalAttDistinctValues 7 - MinorityClassSize 8 - NumberOfClasses 5 - NumberOfFeatures 39 - NumberOfInstances 898 - NumberOfInstancesWithMissingValues 898 - NumberOfMissingValues 22175 - NumberOfNumericFeatures 6 - NumberOfSymbolicFeatures 33 - Name: 2, dtype: object - -We can now filter the data: - - >>> filter = (datasets.NumberOfInstances > 1000) & (datasets.NumberOfFeatures > 5) - >>> filtered_datasets = datasets.loc[filter] - >>> dataset_indices = list(filtered_datasets.index) - >>> print(dataset_indices) # doctest: +SKIP - [3, 6, 12, 14, 16, 18, 20, 21, 22, 23, 24, 26, 28, 30, 32, 36, 38, 44, - ... 5291, 5293, 5295, 5296, 5297, 5301, 5587, 5648, 5889] - -and get a list of dataset indices which can be used in a next step. - -Downloading datasets -~~~~~~~~~~~~~~~~~~~~ - -We can now use the dataset IDs to download all datasets by their IDs. Let's -first look at how to download a single dataset and what can be done with the -dataset object: - -.. code:: python - - >>> dataset_id = 23 - >>> dataset = openml.datasets.get_dataset(dataset_id) - -Properties of the dataset are stored as member variables: - -.. code:: python - >>> print(dataset.__dict__) # doctest: +SKIP - {'upload_date': u'2014-04-06 23:21:03', 'md5_cheksum': u'3149646ecff276abac3e892d1556655f', 'creator': None, 'citation': None, 'tag': [u'study_1', u'study_7', u'uci'], 'version_label': u'1', 'contributor': None, 'paper_url': None, 'original_data_url': None, 'id': 23, 'collection_date': None, 'row_id_attribute': None, 'version': 1, 'data_pickle_file': '/home/matthias/.openml/cache/datasets/23/dataset.pkl', 'default_target_attribute': u'Contraceptive_method_used', 'description': u"**Author**: \n**Source**: Unknown - \n**Please cite**: \n\n1. Title: Contraceptive Method Choice\n \n 2. Sources:\n (a) Origin: This dataset is a subset of the 1987 National Indonesia\n Contraceptive Prevalence Survey\n (b) Creator: Tjen-Sien Lim (limt@stat.wisc.edu)\n (c) Donor: Tjen-Sien Lim (limt@stat.wisc.edu)\n (c) Date: June 7, 1997\n \n 3. Past Usage:\n Lim, T.-S., Loh, W.-Y. & Shih, Y.-S. (1999). A Comparison of\n Prediction Accuracy, Complexity, and Training Time of Thirty-three\n Old and New Classification Algorithms. Machine Learning. Forthcoming.\n (ftp://ftp.stat.wisc.edu/pub/loh/treeprogs/quest1.7/mach1317.pdf or\n (http://www.stat.wisc.edu/~limt/mach1317.pdf)\n \n 4. Relevant Information:\n This dataset is a subset of the 1987 National Indonesia Contraceptive\n Prevalence Survey. The samples are married women who were either not \n pregnant or do not know if they were at the time of interview. The \n problem is to predict the current contraceptive method choice \n (no use, long-term methods, or short-term methods) of a woman based \n on her demographic and socio-economic characteristics.\n \n 5. Number of Instances: 1473\n \n 6. Number of Attributes: 10 (including the class attribute)\n \n 7. Attribute Information:\n \n 1. Wife's age (numerical)\n 2. Wife's education (categorical) 1=low, 2, 3, 4=high\n 3. Husband's education (categorical) 1=low, 2, 3, 4=high\n 4. Number of children ever born (numerical)\n 5. Wife's religion (binary) 0=Non-Islam, 1=Islam\n 6. Wife's now working? (binary) 0=Yes, 1=No\n 7. Husband's occupation (categorical) 1, 2, 3, 4\n 8. Standard-of-living index (categorical) 1=low, 2, 3, 4=high\n 9. Media exposure (binary) 0=Good, 1=Not good\n 10. Contraceptive method used (class attribute) 1=No-use \n 2=Long-term\n 3=Short-term\n \n 8. Missing Attribute Values: None\n\n Information about the dataset\n CLASSTYPE: nominal\n CLASSINDEX: last", 'format': u'ARFF', 'visibility': u'public', 'update_comment': None, 'licence': u'Public', 'name': u'cmc', 'language': None, 'url': u'http://www.openml.org/data/download/23/dataset_23_cmc.arff', 'data_file': '~/.openml/cache/datasets/23/dataset.arff', 'ignore_attributes': None} +~~~~~~~~~~~~ +Key concepts +~~~~~~~~~~~~ -Next, to obtain the data matrix: - -.. code:: python - - >>> X = dataset.get_data() - >>> print(X.shape, X.dtype) - (1473, 10) float32 - -which returns the dataset as a np.ndarray with dtype :python:`np.float32`. -In case the data is sparse, a scipy.sparse.csr matrix is returned. All nominal -variables are encoded as integers, the inverse encoding can be retrieved via: - -.. code:: python - - >>> X, names = dataset.get_data(return_attribute_names=True) - >>> print(names) - ['Wifes_age', 'Wifes_education', 'Husbands_education', 'Number_of_children_ever_born', 'Wifes_religion', 'Wifes_now_working%3F', 'Husbands_occupation', 'Standard-of-living_index', 'Media_exposure', 'Contraceptive_method_used'] - -Most times, having a single data matrix :python:`X` is not enough. Two -useful arguments are :python:`target` and -:python:`return_categorical_indicator`. :python:`target` makes -:meth:`get_data()` return :python:`X` and :python:`y` -seperate; :python:`return_categorical_indicator` makes -:meth:`get_data()` return a boolean array which indicate -which attributes are categorical (and should be one hot encoded if necessary.) - -.. code:: python - - >>> X, y, categorical = dataset.get_data( - ... target=dataset.default_target_attribute, - ... return_categorical_indicator=True) - >>> print(X.shape, y.shape) - (1473, 9) (1473,) - >>> print(categorical) - [False, True, True, False, True, True, True, True, True] - -In case you are working with `scikit-learn -`_, you can use this data right away: - -.. code:: python - - >>> from sklearn import preprocessing, ensemble - >>> enc = preprocessing.OneHotEncoder(categorical_features=categorical) - >>> print(enc) - OneHotEncoder(categorical_features=[False, True, True, False, True, True, True, True, True], - dtype=, handle_unknown='error', - n_values='auto', sparse=True) - >>> X = enc.fit_transform(X).todense() - >>> clf = ensemble.RandomForestClassifier() - >>> clf.fit(X, y) - RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini', - max_depth=None, max_features='auto', max_leaf_nodes=None, - min_impurity_split=1e-07, min_samples_leaf=1, - min_samples_split=2, min_weight_fraction_leaf=0.0, - n_estimators=10, n_jobs=1, oob_score=False, random_state=None, - verbose=0, warm_start=False) - -When you have to retrieve several datasets, you can use the convenience function -:meth:`openml.datasets.get_datasets()`, which downloads all datasets given by -a list of IDs: - - >>> ids = [12, 14, 16, 18, 20, 22] - >>> datasets = openml.datasets.get_datasets(ids) - >>> print(datasets[0].name) - mfeat-factors +OpenML contains several key concepts which it needs to make machine learning +research shareable. A machine learning experiment consists of one or several +**runs**, which describe the performance of an algorithm (called a **flow** in +OpenML), its hyperparameter settings (called a **setup**) on a **task**. A +**Task** is the combination of a **dataset**, a split and an evaluation +metric. In this user guide we will go through listing and exploring existing +**tasks** to actually running machine learning algorithms on them. In a further +user guide we will examine how to search through **datasets** in order to curate +a list of **tasks**. ~~~~~~~~~~~~~~~~~~ Working with tasks ~~~~~~~~~~~~~~~~~~ -#TODO put a link to the OpenML documentation here! Link the Task functions and -the task class - -While datasets provide the most basic information for a machine learning task, -they do not provide enough information for a reproducible machine learning -experiment. A task defines how to split the dataset into a train and test set, -whether to use several disjoint train and test splits (cross-validation) and -whether this should be repeated several times. Also, the task defines a target -metric for which a flow should be optimized. +You can think of a task as an experimentation protocol, describing how to apply +a machine learning model to a dataset in a way that it is comparable with the +results of others (more on how to do that further down).Tasks are containers, +defining which dataset to use, what kind of task we're solving (regression, +classification, clustering, etc...) and which column to predict. Furthermore, +it also describes how to split the dataset into a train and test set, whether +to use several disjoint train and test splits (cross-validation) and whether +this should be repeated several times. Also, the task defines a target metric +for which a flow should be optimized. -Just like datasets, tasks are identified by IDs and can be accessed in three -different ways: +Tasks are identified by IDs and can be accessed in two different ways: 1. In a list providing basic information on all tasks available on OpenML. This function will not download the actual tasks, but will instead download meta data that can be used to filter the tasks and retrieve a set of IDs. -2. By functions only list a subset of all available tasks, restricted either by - their :TODO:`task_type`, :TODO:`tag` or :TODO:`check_for_more`. -3. A single task by its ID. It contains all meta information, the target metric, + We can filter this list, for example, we can only list tasks having a special + tag or only tasks for a specific target such as *supervised classification*. + +2. A single task by its ID. It contains all meta information, the target metric, the splits and an iterator which can be used to access the splits in a useful manner. @@ -257,25 +109,17 @@ You can also read more about tasks in the `OpenML guide >> tasks = openml.tasks.list_tasks(task_type_id=1) -Let's find out more about the datasets: +:meth:`openml.tasks.list_tasks` returns a dictionary of dictionaries, we convert +it into a +`pandas dataframe `_ +to have better visualization and easier access: .. code:: python @@ -291,58 +135,43 @@ Let's find out more about the datasets: 'NumberOfSymbolicFeatures', 'cost_matrix'], dtype='object') -Now we can restrict the tasks to all tasks with the desired resampling strategy: - -# TODO add something about the different resampling strategies implemented! +We can filter the list of tasks to only contain datasets with more than +500 samples, but less than 1000 samples: .. code:: python - >>> filtered_tasks = tasks.query('estimation_procedure == "10-fold Crossvalidation"') - >>> filtered_tasks = list(filtered_tasks.index) - >>> print(filtered_tasks) # doctest: +SKIP - [1, 2, 3, 4, 5, 6, 7, 8, 9, ... 10105, 10106, 10107, 10109, 10111, 13907, 13918] - -Resampling strategies can be found on the `OpenML Website `_ -or programatically as described in `Finding out evaluation strategies and target metrics`_. -Finally, we can check whether there is a task for each dataset that we want to -use in our study. If this is not the case, tasks can be created on the -`OpenML website `_. + >>> filtered_tasks = tasks.query('NumberOfInstances > 500 and NumberOfInstances < 1000') + >>> print(list(filtered_tasks.index)) # doctest: +SKIP + [2, 11, 15, 29, 37, 41, 49, 53, ..., 146597, 146600, 146605] + >>> print(len(filtered_tasks)) # doctest: +SKIP + 210 -The rest of this subsection deals with accessing a list of tasks by tags and -without any restriction. - -A list of tasks, filtered tags, can be retrieved via: +Then, we can further restrict the tasks to all have the same resampling +strategy: .. code:: python - >>> tasks = openml.tasks.list_tasks(tag='study_1') + >>> filtered_tasks = filtered_tasks.query('estimation_procedure == "10-fold Crossvalidation"') + >>> print(list(filtered_tasks.index)) # doctest: +SKIP + [2, 11, 15, 29, 37, 41, 49, 53, ..., 146231, 146238, 146241] + >>> print(len(filtered_tasks)) # doctest: +SKIP + 107 + +Resampling strategies can be found on the `OpenML Website `_. -:meth:`openml.tasks.list_tasks` returns a dict of dictionaries, we will -convert it into a `pandas dataframe `_ -to have better visualization: +Similar to listing tasks by task type, we can list tasks by tags: .. code:: python - >>> import pandas as pd + >>> tasks = openml.tasks.list_tasks(tag='OpenML100') >>> tasks = pd.DataFrame.from_dict(tasks, orient='index') -As before, we have to check whether there is a task for each dataset that we -want to work with. In addition, we have to make sure to use only tasks with the -desired task type: - -#TODO this doesn't look nice, we should have a constant for each known task, -dynamically created by the task type available (but when do we know that we -can savely use the api connector? what to do if we do not have an internet -connection? Maybe have this statically in the program and check from time to -time if there is something new (via a unit test?)?, the same holds true for -the resampling strategies available!) - -.. code:: python - - >>> filter = tasks.task_type == 'Supervised Classification' - >>> filtered_tasks = tasks[filter] - >>> print(len(filtered_tasks)) # doctest: +SKIP - 2599 +*OpenML 100* is a curated list of 100 tasks to start using OpenML. They are all +supervised classification tasks with more than 500 instances and less than 50000 +instances per task. To make things easier, the tasks do not contain highly +unbalanced data and sparse data. However, the tasks include missing values and +categorical features. You can find out more about the *OpenML 100* on +`the OpenML benchmarking page `_. Finally, it is also possible to list all tasks on OpenML with: @@ -350,14 +179,14 @@ Finally, it is also possible to list all tasks on OpenML with: >>> tasks = openml.tasks.list_tasks() >>> print(len(tasks)) # doctest: +SKIP - 29757 + 46067 Downloading tasks ~~~~~~~~~~~~~~~~~ -Downloading tasks works similar to downloading datasets. We provide two -functions for this, one which downloads only a single task by its ID, -and one which takes a list of IDs and downloads all of these tasks: +We provide two functions to download tasks, one which downloads only a single +task by its ID, and one which takes a list of IDs and downloads all of these +tasks: .. code:: python @@ -390,89 +219,23 @@ Properties of the task are stored as member variables: 'task_type': 'Supervised Classification', 'task_type_id': 1} -And with a list of task IDs: +And: .. code:: python - >>> ids = [12, 14, 16, 18, 20, 22] + >>> ids = [2, 11, 15, 29, 37, 41, 49, 53] >>> tasks = openml.tasks.get_tasks(ids) >>> pprint(tasks[0]) # doctest: +SKIP -~~~~~~~~~~~~~~~~~~~~~~~ -Finding out tasks types -~~~~~~~~~~~~~~~~~~~~~~~ - -Not yet supported by the API. Please use the OpenML website. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Finding out evaluation strategies and target metrics -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Not yet supported by the API. Please use the OpenML website. - -~~~~~~~~~~~~~~~ -Using the cache -~~~~~~~~~~~~~~~ - -Downloading all datasets, tasks and split every time a get function is called -would prohibit a user to interact with the API in an exploratory manner. -OpenML is designed in a way that certain entities are immutable once created. -This allows the python package to cache datasets, tasks, splits and runs locally -for fast retrieval. Another benefit is that the API can be used normally on a -compute cluster without internet access (:ref:`see below`). - -Currently, the following objects are cached: - -* datasets - * dataset arff. In order to reduce parsing time, the data is serialized to - disk in a binary format (using the `pickle library `_. - * dataset descriptions - * more? -* tasks - * task description - * split arff. TODO are they cached? -* runs - * run description - -Run predictions are not cached yet. Flow ojects cannot yet be downloaded and are -therefore not cached. - -Configuring the cache -~~~~~~~~~~~~~~~~~~~~~ - -Configuring the cache works as described in the subsection `Connecting to the OpenML server`_: -It can be done either through the API: - -.. code:: python - - >>> openml.config.set_cache_directory(os.path.expanduser('~/.openml/cache')) - -or the config file: - -.. code:: bash - - cachedir = '~/.openml/cache' - - -Clearing the cache -~~~~~~~~~~~~~~~~~~ - -Currently, there is no programmatic way to interact with the cache and we do not -plan to implement one. If you have any use case for this, please open an issue -on the `issue tracker `_. - -# TODO check that the cache is in a consistent state! -In case the cache gets too large, you can manually delete unnecessary files. -Make sure that you always delete a complete entity, for example the whole -directory caching a dataset named after the datasets ID. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Working with Flows and Runs -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~ +Creating runs +~~~~~~~~~~~~~ -Tasks and datasets allow us to download all information to run an experiment -locally. In order to upload and share results of such an experiment we need -the concepts of flows and runs. +In order to upload and share results of running a machine learning algorithm +on a task, we need to create an :class:`~openml.OpenMLRun`. A run object can +be created by running a :class:`~openml.OpenMLFlow` or a scikit-learn compatible +model on a task. We will focus on the simpler example of running a +scikit-learn model. Flows are descriptions of something runable which does the machine learning. A flow contains all information to set up the necessary machine learning @@ -499,6 +262,80 @@ Running a model >>> task = openml.tasks.get_task(12) >>> run = openml.runs.run_model_on_task(task, model) >>> pprint(vars(run), depth=2) # doctest: +SKIP + {'data_content': [...], + 'dataset_id': 12, + 'error_message': None, + 'evaluations': None, + 'flow': None, + 'flow_id': 7257, + 'flow_name': None, + 'fold_evaluations': defaultdict(. at 0x7fb88981b9d8>, + {'predictive_accuracy': defaultdict(, + {0: {0: 0.94499999999999995, + 1: 0.94499999999999995, + 2: 0.94499999999999995, + 3: 0.96499999999999997, + 4: 0.92500000000000004, + 5: 0.96499999999999997, + 6: 0.94999999999999996, + 7: 0.96999999999999997, + 8: 0.93999999999999995, + 9: 0.95499999999999996}}), + 'usercpu_time_millis': defaultdict(, + {0: {0: 110.4880920000042, + 1: 105.7469440000034, + 2: 107.4153629999941, + 3: 105.1104170000059, + 4: 104.02388900000403, + 5: 105.17172800000196, + 6: 109.00792000001047, + 7: 107.49670599999206, + 8: 107.34138000000115, + 9: 104.78881499999915}}), + 'usercpu_time_millis_testing': defaultdict(, + {0: {0: 3.6470320000034917, + 1: 3.5307810000020368, + 2: 3.5432540000002177, + 3: 3.5460690000022055, + 4: 3.5634600000022942, + 5: 3.906016000001955, + 6: 3.6680000000046675, + 7: 3.643865999997331, + 8: 3.4515420000005292, + 9: 3.461469000001216}}), + 'usercpu_time_millis_training': defaultdict(, + {0: {0: 106.84106000000071, + 1: 102.21616300000136, + 2: 103.87210899999388, + 3: 101.56434800000369, + 4: 100.46042900000174, + 5: 101.26571200000001, + 6: 105.3399200000058, + 7: 103.85283999999473, + 8: 103.88983800000062, + 9: 101.32734599999793}})}), + 'model': RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini', + max_depth=None, max_features='auto', max_leaf_nodes=None, + min_impurity_split=1e-07, min_samples_leaf=1, + min_samples_split=2, min_weight_fraction_leaf=0.0, + n_estimators=10, n_jobs=1, oob_score=False, random_state=43934, + verbose=0, warm_start=False), + 'output_files': None, + 'parameter_settings': [...], + 'predictions_url': None, + 'run_id': None, + 'sample_evaluations': None, + 'setup_id': None, + 'setup_string': None, + 'tags': [...], + 'task': None, + 'task_evaluation_measure': None, + 'task_id': 12, + 'task_type': None, + 'trace_attributes': None, + 'trace_content': None, + 'uploader': None, + 'uploader_name': None} So far the run is only available locally. By calling the publish function, the run is send to the OpenML server: @@ -506,14 +343,14 @@ run is send to the OpenML server: .. code:: python >>> run.publish() # doctest: +SKIP - # What happens here? What should it return? + We can now also inspect the flow object which was automatically created: .. code:: python >>> flow = openml.flows.get_flow(run.flow_id) - >>> pprint(vars(flow), depth=2) # doctest: +SKIP + >>> pprint(vars(flow), depth=1) # doctest: +SKIP {'binary_format': None, 'binary_md5': None, 'binary_url': None, @@ -523,7 +360,7 @@ We can now also inspect the flow object which was automatically created: 'dependencies': 'sklearn==0.18.2\nnumpy>=1.6.1\nscipy>=0.9', 'description': 'Automatically created scikit-learn flow.', 'external_version': 'openml==0.6.0,sklearn==0.18.2', - 'flow_id': 7245, + 'flow_id': 7257, 'language': 'English', 'model': RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini', max_depth=None, max_features='auto', max_leaf_nodes=None, @@ -534,23 +371,18 @@ We can now also inspect the flow object which was automatically created: 'name': 'sklearn.ensemble.forest.RandomForestClassifier', 'parameters': OrderedDict([...]), 'parameters_meta_info': OrderedDict([...]), - 'tags': ['openml-python', - 'python', - 'scikit-learn', - 'sklearn', - 'sklearn_0.18.2'], - 'upload_date': '2017-10-06T14:54:38', - 'uploader': '86', - 'version': '28'} - -Retrieving results from OpenML -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# TODO - - - - + 'tags': [...], + 'upload_date': '2017-10-09T10:20:40', + 'uploader': '1159', + 'version': '29'} +Advanced topics +~~~~~~~~~~~~~~~ +We are working on tutorials for the following topics: +* Querying datasets +* Uploading datasets +* Creating tasks +* Working offline +* Analyzing large amounts of results diff --git a/examples/OpenML_Tutorial.ipynb b/examples/OpenML_Tutorial.ipynb index dcc7aedec..a8ec24e78 100644 --- a/examples/OpenML_Tutorial.ipynb +++ b/examples/OpenML_Tutorial.ipynb @@ -23,14 +23,18 @@ ] }, { - "cell_type": "raw", + "cell_type": "markdown", "metadata": { - "collapsed": true + "slideshow": { + "slide_type": "slide" + } }, "source": [ - "# Install OpenML (developer version)\n", - "# 'pip install openml' coming up (october 2017) \n", - "pip install git+https://github.com/openml/openml-python.git@develop" + "# Installation\n", + "\n", + "* Up to now: `pip install git+https://github.com/openml/openml-python.git@develop`\n", + "* In the future: `pip install openml`\n", + "* Check out the installation guide: [https://openml.github.io/openml-python/stable/#installation](https://openml.github.io/openml-python/stable/#installation)" ] }, { @@ -842,8 +846,9 @@ ], "source": [ "X, y, attribute_names = dataset.get_data(\n", - " target=dataset.default_target_attribute, \n", - " return_attribute_names=True)\n", + " target=dataset.default_target_attribute,\n", + " return_attribute_names=True,\n", + ")\n", "eeg = pd.DataFrame(X, columns=attribute_names)\n", "eeg['class'] = y\n", "print(eeg[:10])" @@ -989,7 +994,8 @@ "dataset = oml.datasets.get_dataset(10)\n", "X, y, categorical = dataset.get_data(\n", " target=dataset.default_target_attribute,\n", - " return_categorical_indicator=True)\n", + " return_categorical_indicator=True,\n", + ")\n", "print(\"Categorical features: %s\" % categorical)\n", "enc = preprocessing.OneHotEncoder(categorical_features=categorical)\n", "X = enc.fit_transform(X)\n", @@ -1547,7 +1553,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.0" + "version": "3.6.2" } }, "nbformat": 4, diff --git a/openml/__init__.py b/openml/__init__.py index b8c1fa6a8..d34f1bab6 100644 --- a/openml/__init__.py +++ b/openml/__init__.py @@ -28,6 +28,7 @@ from .runs import OpenMLRun from .tasks import OpenMLTask, OpenMLSplit from .flows import OpenMLFlow +from .evaluations import OpenMLEvaluation from .__version__ import __version__ diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 043759559..93f0ed2f1 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -7,7 +7,8 @@ import xmltodict from . import config -from .exceptions import OpenMLServerError, OpenMLServerException +from .exceptions import (OpenMLServerError, OpenMLServerException, + OpenMLServerNoResult) def _perform_api_call(call, data=None, file_dictionary=None, @@ -94,7 +95,7 @@ def _read_url_files(url, data=None, file_dictionary=None, file_elements=None): # 'gzip,deflate' response = requests.post(url, data=data, files=file_elements) if response.status_code != 200: - raise _parse_server_exception(response) + raise _parse_server_exception(response, url=url) if 'Content-Encoding' not in response.headers or \ response.headers['Content-Encoding'] != 'gzip': warnings.warn('Received uncompressed content from OpenML for %s.' % url) @@ -116,14 +117,14 @@ def _read_url(url, data=None): response = requests.post(url, data=data) if response.status_code != 200: - raise _parse_server_exception(response) + raise _parse_server_exception(response, url=url) if 'Content-Encoding' not in response.headers or \ response.headers['Content-Encoding'] != 'gzip': warnings.warn('Received uncompressed content from OpenML for %s.' % url) return response.text -def _parse_server_exception(response): +def _parse_server_exception(response, url=None): # OpenML has a sopisticated error system # where information about failures is provided. try to parse this try: @@ -138,4 +139,13 @@ def _parse_server_exception(response): additional = None if 'oml:additional_information' in server_exception['oml:error']: additional = server_exception['oml:error']['oml:additional_information'] - return OpenMLServerException(code, message, additional) + if code in [372, 512, 500, 482, 542, 674]: # datasets, + # 512 for runs, 372 for datasets, 500 for flows + # 482 for tasks, 542 for evaluations, 674 for setups + return OpenMLServerNoResult(code, message, additional) + return OpenMLServerException( + code=code, + message=message, + additional=additional, + url=url + ) diff --git a/openml/config.py b/openml/config.py index 192b5fcaa..cb79da653 100644 --- a/openml/config.py +++ b/openml/config.py @@ -6,6 +6,7 @@ from six import StringIO from six.moves import configparser +from six.moves.urllib_parse import urlparse logger = logging.getLogger(__name__) @@ -13,10 +14,23 @@ format='[%(levelname)s] [%(asctime)s:%(name)s] %(' 'message)s', datefmt='%H:%M:%S') +# Default values! +_defaults = { + 'apikey': None, + 'server': "https://www.openml.org/api/v1/xml", + 'verbosity': 0, + 'cachedir': os.path.expanduser('~/.openml/cache'), + 'avoid_duplicate_runs': 'True', +} + config_file = os.path.expanduser('~/.openml/config') -server = "https://www.openml.org/api/v1/xml" + +# Default values are actually added here in the _setup() function which is +# called at the end of this module +server = "" apikey = "" -cachedir = "" +# The current cache directory (without the server name) +cache_directory = "" def _setup(): @@ -26,12 +40,11 @@ def _setup(): key and server can be set by the user simply using openml.config.apikey = THEIRKEY openml.config.server = SOMESERVER - The cache dir needs to be set up calling set_cache_directory - because it needs some setup. We could also make it a property but that's less clear. """ global apikey global server + global cache_directory global avoid_duplicate_runs # read config file, create cache directory try: @@ -42,52 +55,15 @@ def _setup(): config = _parse_config() apikey = config.get('FAKE_SECTION', 'apikey') server = config.get('FAKE_SECTION', 'server') - cache_dir = config.get('FAKE_SECTION', 'cachedir') + cache_directory = os.path.expanduser(config.get('FAKE_SECTION', 'cachedir')) avoid_duplicate_runs = config.getboolean('FAKE_SECTION', 'avoid_duplicate_runs') - set_cache_directory(cache_dir) - - -def set_cache_directory(cachedir): - """Set module-wide cache directory. - - Sets the cache directory into which to download datasets, tasks etc. - - Parameters - ---------- - cachedir : string - Path to use as cache directory. - - See also - -------- - get_cache_directory - """ - - global _cachedir - _cachedir = cachedir - - # Set up the cache directories - dataset_cache_dir = os.path.join(cachedir, "datasets") - task_cache_dir = os.path.join(cachedir, "tasks") - run_cache_dir = os.path.join(cachedir, 'runs') - lock_dir = os.path.join(cachedir, 'locks') - - for dir_ in [ - cachedir, dataset_cache_dir, task_cache_dir, run_cache_dir, lock_dir, - ]: - if not os.path.exists(dir_) and not os.path.isdir(dir_): - os.mkdir(dir_) def _parse_config(): """Parse the config file, set up defaults. """ - defaults = {'apikey': apikey, - 'server': server, - 'verbosity': 0, - 'cachedir': os.path.expanduser('~/.openml/cache'), - 'avoid_duplicate_runs': 'True'} - config = configparser.RawConfigParser(defaults=defaults) + config = configparser.RawConfigParser(defaults=_defaults) if not os.path.exists(config_file): # Create an empty config file if there was none so far @@ -106,8 +82,7 @@ def _parse_config(): config_file_.seek(0) config.readfp(config_file_) except OSError as e: - logging.info("Error opening file %s: %s" % - config_file, e.message) + logging.info("Error opening file %s: %s", config_file, e.message) return config @@ -119,13 +94,38 @@ def get_cache_directory(): cachedir : string The current cache directory. + """ + url_suffix = urlparse(server).netloc + reversed_url_suffix = '/'.join(url_suffix.split('.')[::-1]) + if not cache_directory: + _cachedir = _defaults(cache_directory) + else: + _cachedir = cache_directory + _cachedir = os.path.join(_cachedir, reversed_url_suffix) + return _cachedir + + +def set_cache_directory(cachedir): + """Set module-wide cache directory. + + Sets the cache directory into which to download datasets, tasks etc. + + Parameters + ---------- + cachedir : string + Path to use as cache directory. + See also -------- - set_cache_directory + get_cache_directory """ - return _cachedir + + global cache_directory + cache_directory = cachedir -__all__ = ["set_cache_directory", 'get_cache_directory'] +__all__ = [ + 'get_cache_directory', 'set_cache_directory' +] _setup() diff --git a/openml/datasets/data_feature.py b/openml/datasets/data_feature.py index 0254d4624..51b132f1c 100644 --- a/openml/datasets/data_feature.py +++ b/openml/datasets/data_feature.py @@ -1,3 +1,4 @@ +import six class OpenMLDataFeature(object): """Data Feature (a.k.a. Attribute) object. @@ -16,21 +17,30 @@ class OpenMLDataFeature(object): """ LEGAL_DATA_TYPES = ['nominal', 'numeric', 'string', 'date'] - def __init__(self, index, name, data_type, nominal_values, number_missing_values): + def __init__(self, index, name, data_type, nominal_values, + number_missing_values): if type(index) != int: raise ValueError('Index is of wrong datatype') if data_type not in self.LEGAL_DATA_TYPES: - raise ValueError('data type should be in %s, found: %s' %(str(self.LEGAL_DATA_TYPES),data_type)) + raise ValueError('data type should be in %s, found: %s' % + (str(self.LEGAL_DATA_TYPES), data_type)) if nominal_values is not None and type(nominal_values) != list: raise ValueError('Nominal_values is of wrong datatype') if type(number_missing_values) != int: raise ValueError('number_missing_values is of wrong datatype') self.index = index - self.name = str(name) + # In case of python version lower than 3, change the default ASCII encoder. + if six.PY2: + self.name = str(name.encode('utf8')) + else: + self.name = str(name) self.data_type = str(data_type) self.nominal_values = nominal_values self.number_missing_values = number_missing_values def __str__(self): - return "[%d - %s (%s)]" %(self.index, self.name, self.data_type) \ No newline at end of file + return "[%d - %s (%s)]" % (self.index, self.name, self.data_type) + + def _repr_pretty_(self, pp, cycle): + pp.text(str(self)) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index e8d6e8778..f25557783 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -3,7 +3,6 @@ import logging import os import six -import sys import arff @@ -14,7 +13,7 @@ from .data_feature import OpenMLDataFeature from ..exceptions import PyOpenMLError -from .._api_calls import _perform_api_call +import openml._api_calls logger = logging.getLogger(__name__) @@ -26,9 +25,9 @@ class OpenMLDataset(object): Parameters ---------- - name : string + name : str Name of the dataset - description : string + description : str Description of the dataset FIXME : which of these do we actually nee? """ @@ -82,22 +81,20 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None, feature = OpenMLDataFeature(int(xmlfeature['oml:index']), xmlfeature['oml:name'], xmlfeature['oml:data_type'], - None, #todo add nominal values (currently not in database) + None, # todo add nominal values (currently not in database) int(xmlfeature.get('oml:number_of_missing_values', 0))) if idx != feature.index: raise ValueError('Data features not provided in right order') self.features[feature.index] = feature - if qualities is not None: - self.qualities = {} - for idx, xmlquality in enumerate(qualities['oml:quality']): - name = xmlquality['oml:name'] - value = xmlquality['oml:value'] - self.qualities[name] = value + self.qualities = _check_qualities(qualities) if data_file is not None: if self._data_features_supported(): - self.data_pickle_file = data_file.replace('.arff', '.pkl') + if six.PY2: + self.data_pickle_file = data_file.replace('.arff', '.pkl.py2') + else: + self.data_pickle_file = data_file.replace('.arff', '.pkl.py3') if os.path.exists(self.data_pickle_file): logger.debug("Data pickle file already exists.") @@ -127,13 +124,37 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None, with open(self.data_pickle_file, "wb") as fh: pickle.dump((X, categorical, attribute_names), fh, -1) logger.debug("Saved dataset %d: %s to file %s" % - (self.dataset_id, self.name, self.data_pickle_file)) + (int(self.dataset_id or -1), self.name, self.data_pickle_file)) + + def push_tag(self, tag): + """Annotates this data set with a tag on the server. + + Parameters + ---------- + tag : str + Tag to attach to the dataset. + """ + data = {'data_id': self.dataset_id, 'tag': tag} + openml._api_calls._perform_api_call("/data/tag", data=data) + + def remove_tag(self, tag): + """Removes a tag from this dataset on the server. + + Parameters + ---------- + tag : str + Tag to attach to the dataset. + """ + data = {'data_id': self.dataset_id, 'tag': tag} + openml._api_calls._perform_api_call("/data/untag", data=data) def __eq__(self, other): if type(other) != OpenMLDataset: return False - elif self.id == other._id or \ - (self.name == other._name and self.version == other._version): + elif ( + self.dataset_id == other.dataset_id + or (self.name == other._name and self.version == other._version) + ): return True else: return False @@ -184,10 +205,12 @@ def decode_arff(fh): with io.open(filename, encoding='utf8') as fh: return decode_arff(fh) - def get_data(self, target=None, target_dtype=int, include_row_id=False, + def get_data(self, target=None, + include_row_id=False, include_ignore_attributes=False, return_categorical_indicator=False, - return_attribute_names=False): + return_attribute_names=False + ): """Returns dataset content as numpy arrays / sparse matrices. Parameters @@ -201,7 +224,10 @@ def get_data(self, target=None, target_dtype=int, include_row_id=False, rval = [] if not self._data_features_supported(): - raise PyOpenMLError('Dataset not compatible, PyOpenML cannot handle string features') + raise PyOpenMLError( + 'Dataset %d not compatible, PyOpenML cannot handle string ' + 'features' % self.dataset_id + ) path = self.data_pickle_file if not os.path.exists(path): @@ -225,7 +251,10 @@ def get_data(self, target=None, target_dtype=int, include_row_id=False, if not self.ignore_attributes: pass else: - to_exclude.extend(self.ignore_attributes) + if isinstance(self.ignore_attributes, six.string_types): + to_exclude.append(self.ignore_attributes) + else: + to_exclude.extend(self.ignore_attributes) if len(to_exclude) > 0: logger.info("Going to remove the following attributes:" @@ -241,9 +270,23 @@ def get_data(self, target=None, target_dtype=int, include_row_id=False, rval.append(data) else: if isinstance(target, six.string_types): - target = [target] + if ',' in target: + target = target.split(',') + else: + target = [target] targets = np.array([True if column in target else False for column in attribute_names]) + if np.sum(targets) > 1: + raise NotImplementedError( + "Number of requested targets %d is not implemented." % + np.sum(targets) + ) + target_categorical = [ + cat for cat, column in + six.moves.zip(categorical, attribute_names) + if column in target + ] + target_dtype = int if target_categorical[0] else float try: x = data[:, ~targets] @@ -315,7 +358,6 @@ def retrieve_class_labels(self, target_name='class'): else: return None - def get_features_by_type(self, data_type, exclude=None, exclude_ignore_attributes=True, exclude_row_id_attribute=True): @@ -342,13 +384,17 @@ def get_features_by_type(self, data_type, exclude=None, result : list a list of indices that have the specified data type ''' - assert data_type in OpenMLDataFeature.LEGAL_DATA_TYPES, "Illegal feature type requested" + if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES: + raise TypeError("Illegal feature type requested") if self.ignore_attributes is not None: - assert type(self.ignore_attributes) is list, "ignore_attributes should be a list" + if not isinstance(self.ignore_attributes, list): + raise TypeError("ignore_attributes should be a list") if self.row_id_attribute is not None: - assert type(self.row_id_attribute) is str, "row id attribute should be a str" + if not isinstance(self.row_id_attribute, six.string_types): + raise TypeError("row id attribute should be a str") if exclude is not None: - assert type(exclude) is list, "Exclude should be a list" + if not isinstance(exclude, list): + raise TypeError("Exclude should be a list") # assert all(isinstance(elem, str) for elem in exclude), "Exclude should be a list of strings" to_exclude = [] if exclude is not None: @@ -377,11 +423,7 @@ def publish(self): Returns ------- - return_code : int - Return code from server - - return_value : string - xml return from server + self """ file_elements = {'description': self._to_xml()} @@ -390,8 +432,11 @@ def publish(self): if self.data_file is not None: file_dictionary['dataset'] = self.data_file - return_value = _perform_api_call("/data/", file_dictionary=file_dictionary, - file_elements=file_elements) + return_value = openml._api_calls._perform_api_call( + "/data/", + file_dictionary=file_dictionary, + file_elements=file_elements, + ) self.dataset_id = int(xmltodict.parse(return_value)['oml:upload_data_set']['oml:id']) return self @@ -401,7 +446,7 @@ def _to_xml(self): Returns ------- - xml_dataset : string + xml_dataset : str XML description of the data. """ xml_dataset = (' 0): raise ValueError('Argument \'version\' should be a non-empty string') - xml_response = _perform_api_call("flow/exists", - data={'name': name, 'external_version': external_version}) + xml_response = openml._api_calls._perform_api_call( + "flow/exists", + data={'name': name, 'external_version': external_version}, + ) result_dict = xmltodict.parse(xml_response) flow_id = int(result_dict['oml:flow_exists']['oml:id']) @@ -104,16 +126,16 @@ def flow_exists(name, external_version): return False -def _list_flows(api_call): - # TODO add proper error handling here! - xml_string = _perform_api_call(api_call) +def __list_flows(api_call): + + xml_string = openml._api_calls._perform_api_call(api_call) flows_dict = xmltodict.parse(xml_string, force_list=('oml:flow',)) # Minimalistic check if the XML is useful assert type(flows_dict['oml:flows']['oml:flow']) == list, \ type(flows_dict['oml:flows']) assert flows_dict['oml:flows']['@xmlns:oml'] == \ - 'http://openml.org/openml', flows_dict['oml:flows']['@xmlns:oml'] + 'http://openml.org/openml', flows_dict['oml:flows']['@xmlns:oml'] flows = dict() for flow_ in flows_dict['oml:flows']['oml:flow']: @@ -180,20 +202,20 @@ def assert_flows_equal(flow1, flow2, # Tags aren't directly created by the server, # but the uploader has no control over them! 'tags'] - ignored_by_python_API = ['binary_url', 'binary_format', 'binary_md5', + ignored_by_python_api = ['binary_url', 'binary_format', 'binary_md5', 'model'] for key in set(flow1.__dict__.keys()).union(flow2.__dict__.keys()): - if key in generated_by_the_server + ignored_by_python_API: + if key in generated_by_the_server + ignored_by_python_api: continue attr1 = getattr(flow1, key, None) attr2 = getattr(flow2, key, None) if key == 'components': for name in set(attr1.keys()).union(attr2.keys()): - if not name in attr1: + if name not in attr1: raise ValueError('Component %s only available in ' 'argument2, but not in argument1.' % name) - if not name in attr2: + if name not in attr2: raise ValueError('Component %s only available in ' 'argument2, but not in argument1.' % name) assert_flows_equal(attr1[name], attr2[name], diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 43513a293..9e9697480 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -2,6 +2,7 @@ import io import json import os +import shutil import sys import time import warnings @@ -14,13 +15,13 @@ import openml import openml.utils -from ..exceptions import PyOpenMLError +import openml._api_calls +from ..exceptions import PyOpenMLError, OpenMLServerNoResult from .. import config from ..flows import sklearn_to_flow, get_flow, flow_exists, _check_n_jobs, \ _copy_server_fields from ..setups import setup_exists, initialize_model from ..exceptions import OpenMLCacheException, OpenMLServerException -from .._api_calls import _perform_api_call from .run import OpenMLRun, _get_version_information from .trace import OpenMLRunTrace, OpenMLTraceIteration @@ -28,10 +29,12 @@ # _get_version_info, _get_dict and _create_setup_string are in run.py to avoid # circular imports +RUNS_CACHE_DIR_NAME = 'runs' + def run_model_on_task(task, model, avoid_duplicate_runs=True, flow_tags=None, seed=None): - """See ``run_flow_on_task for a documentation.""" + """See ``run_flow_on_task for a documentation``.""" flow = sklearn_to_flow(model) @@ -89,8 +92,7 @@ def run_flow_on_task(task, flow, avoid_duplicate_runs=True, flow_tags=None, dataset = task.get_dataset() - class_labels = task.class_labels - if class_labels is None: + if task.class_labels is None: raise ValueError('The task has no class labels. This method currently ' 'only works for tasks with class labels.') @@ -98,9 +100,10 @@ def run_flow_on_task(task, flow, avoid_duplicate_runs=True, flow_tags=None, tags = ['openml-python', run_environment[1]] # execute the run - res = _run_task_get_arffcontent(flow.model, task, class_labels) + res = _run_task_get_arffcontent(flow.model, task) - if flow.flow_id is None: + # in case the flow not exists, we will get a "False" back (which can be + if not isinstance(flow.flow_id, int) or flow_id == False: _publish_flow_if_necessary(flow) run = OpenMLRun(task_id=task.task_id, flow_id=flow.flow_id, @@ -150,7 +153,7 @@ def get_run_trace(run_id): openml.runs.OpenMLTrace """ - trace_xml = _perform_api_call('run/trace/%d' % run_id) + trace_xml = openml._api_calls._perform_api_call('run/trace/%d' % run_id) run_trace = _create_trace_from_description(trace_xml) return run_trace @@ -223,31 +226,37 @@ def initialize_model_from_trace(run_id, repeat, fold, iteration=None): def _run_exists(task_id, setup_id): - ''' - Checks whether a task/setup combination is already present on the server. + """Checks whether a task/setup combination is already present on the server. - :param task_id: int - :param setup_id: int - :return: List of run ids iff these already exists on the server, False otherwise - ''' + Parameters + ---------- + task_id: int + + setup_id: int + + Returns + ------- + Set run ids for runs where flow setup_id was run on task_id. Empty + set if it wasn't run yet. + """ if setup_id <= 0: # openml setups are in range 1-inf - return False + return set() try: result = list_runs(task=[task_id], setup=[setup_id]) if len(result) > 0: return set(result.keys()) else: - return False + return set() except OpenMLServerException as exception: # error code 512 implies no results. This means the run does not exist yet assert(exception.code == 512) - return False + return set() def _get_seeded_model(model, seed=None): - '''Sets all the non-seeded components of a model with a seed. + """Sets all the non-seeded components of a model with a seed. Models that are already seeded will maintain the seed. In this case, only integer seeds are allowed (An exception is thrown when a RandomState was used as seed) @@ -265,7 +274,7 @@ def _get_seeded_model(model, seed=None): model : sklearn model a version of the model where all (sub)components have a seed - ''' + """ def _seed_current_object(current_value): if isinstance(current_value, int): # acceptable behaviour @@ -359,8 +368,7 @@ def _prediction_to_row(rep_no, fold_no, sample_no, row_id, correct_label, return arff_line -# JvR: why is class labels a parameter? could be removed and taken from task object, right? -def _run_task_get_arffcontent(model, task, class_labels): +def _run_task_get_arffcontent(model, task): def _prediction_to_probabilities(y, model_classes): # y: list or numpy array of predictions @@ -373,18 +381,17 @@ def _prediction_to_probabilities(y, model_classes): result[obs][array_idx] = 1.0 return result - X, Y = task.get_X_and_y() arff_datacontent = [] arff_tracecontent = [] # stores fold-based evaluation measures. In case of a sample based task, # this information is multiple times overwritten, but due to the ordering # of tne loops, eventually it contains the information based on the full # dataset size - user_defined_measures_fold = defaultdict(lambda: defaultdict(dict)) + user_defined_measures_per_fold = defaultdict(lambda: defaultdict(dict)) # stores sample-based evaluation measures (sublevel of fold-based) # will also be filled on a non sample-based task, but the information # is the same as the fold-based measures, and disregarded in that case - user_defined_measures_sample = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) + user_defined_measures_per_sample = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) # sys.version_info returns a tuple, the following line compares the entry of tuples # https://docs.python.org/3.6/reference/expressions.html#value-comparisons @@ -397,80 +404,18 @@ def _prediction_to_probabilities(y, model_classes): for fold_no in range(num_folds): for sample_no in range(num_samples): model_fold = sklearn.base.clone(model, safe=True) - train_indices, test_indices = task.get_train_test_split_indices(repeat=rep_no, - fold=fold_no, - sample=sample_no) - trainX = X[train_indices] - trainY = Y[train_indices] - testX = X[test_indices] - testY = Y[test_indices] - - try: - # for measuring runtime. Only available since Python 3.3 - if can_measure_runtime: - modelfit_starttime = time.process_time() - model_fold.fit(trainX, trainY) - - if can_measure_runtime: - modelfit_duration = (time.process_time() - modelfit_starttime) * 1000 - user_defined_measures_sample['usercpu_time_millis_training'][rep_no][fold_no][sample_no] = modelfit_duration - user_defined_measures_fold['usercpu_time_millis_training'][rep_no][fold_no] = modelfit_duration - except AttributeError as e: - # typically happens when training a regressor on classification task - raise PyOpenMLError(str(e)) - - # extract trace, if applicable - if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): - arff_tracecontent.extend(_extract_arfftrace(model_fold, rep_no, fold_no)) - - # search for model classes_ (might differ depending on modeltype) - # first, pipelines are a special case (these don't have a classes_ - # object, but rather borrows it from the last step. We do this manually, - # because of the BaseSearch check) - if isinstance(model_fold, sklearn.pipeline.Pipeline): - used_estimator = model_fold.steps[-1][-1] - else: - used_estimator = model_fold + res =_run_model_on_fold(model_fold, task, rep_no, fold_no, sample_no, can_measure_runtime) + arff_datacontent_fold, arff_tracecontent_fold, user_defined_measures_fold, model_fold = res - if isinstance(used_estimator, sklearn.model_selection._search.BaseSearchCV): - model_classes = used_estimator.best_estimator_.classes_ - else: - model_classes = used_estimator.classes_ - - if can_measure_runtime: - modelpredict_starttime = time.process_time() - - PredY = model_fold.predict(testX) - try: - ProbaY = model_fold.predict_proba(testX) - except AttributeError: - ProbaY = _prediction_to_probabilities(PredY, list(model_classes)) - - # add client-side calculated metrics. These might be used on the server as consistency check - def _calculate_local_measure(sklearn_fn, openml_name): - user_defined_measures_fold[openml_name][rep_no][fold_no] = \ - sklearn_fn(testY, PredY) - user_defined_measures_sample[openml_name][rep_no][fold_no][sample_no] = \ - sklearn_fn(testY, PredY) - - _calculate_local_measure(sklearn.metrics.accuracy_score, 'predictive_accuracy') - - if can_measure_runtime: - modelpredict_duration = (time.process_time() - modelpredict_starttime) * 1000 - user_defined_measures_fold['usercpu_time_millis_testing'][rep_no][fold_no] = modelpredict_duration - user_defined_measures_fold['usercpu_time_millis'][rep_no][fold_no] = modelfit_duration + modelpredict_duration - user_defined_measures_sample['usercpu_time_millis_testing'][rep_no][fold_no][sample_no] = modelpredict_duration - user_defined_measures_sample['usercpu_time_millis'][rep_no][fold_no][sample_no] = modelfit_duration + modelpredict_duration - - if ProbaY.shape[1] != len(class_labels): - warnings.warn("Repeat %d Fold %d: estimator only predicted for %d/%d classes!" %(rep_no, fold_no, ProbaY.shape[1], len(class_labels))) - - for i in range(0, len(test_indices)): - arff_line = _prediction_to_row(rep_no, fold_no, sample_no, - test_indices[i], class_labels[testY[i]], - PredY[i], ProbaY[i], class_labels, model_classes) - arff_datacontent.append(arff_line) + arff_datacontent.extend(arff_datacontent_fold) + arff_tracecontent.extend(arff_tracecontent_fold) + + for measure in user_defined_measures_fold: + user_defined_measures_per_fold[measure][rep_no][fold_no] = user_defined_measures_fold[measure] + user_defined_measures_per_sample[measure][rep_no][fold_no][sample_no] = user_defined_measures_fold[measure] + # Note that we need to use a fitted model (i.e., model_fold, and not model) here, + # to ensure it contains the hyperparameter data (in cv_results_) if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): # arff_tracecontent is already set arff_trace_attributes = _extract_arfftrace_attributes(model_fold) @@ -481,8 +426,136 @@ def _calculate_local_measure(sklearn_fn, openml_name): return arff_datacontent, \ arff_tracecontent, \ arff_trace_attributes, \ - user_defined_measures_fold, \ - user_defined_measures_sample + user_defined_measures_per_fold, \ + user_defined_measures_per_sample + + +def _run_model_on_fold(model, task, rep_no, fold_no, sample_no, can_measure_runtime): + """Internal function that executes a model on a fold (and possibly + subsample) of the dataset. It returns the data that is necessary + to construct the OpenML Run object (potentially over more than + one folds). Is used by run_task_get_arff_content. Do not use this + function unless you know what you are doing. + + Parameters + ---------- + model : sklearn model + The UNTRAINED model to run + task : OpenMLTask + The task to run the model on + rep_no : int + The repeat of the experiment (0-based; in case of 1 time CV, + always 0) + fold_no : int + The fold nr of the experiment (0-based; in case of holdout, + always 0) + sample_no : int + In case of learning curves, the index of the subsample (0-based; + in case of no learning curve, always 0) + can_measure_runtime : bool + Wether we are allowed to measure runtime (requires: Single node + computation and Python >= 3.3) + + Returns + ------- + arff_datacontent : List[List] + Arff representation (list of lists) of the predictions that were + generated by this fold (for putting in predictions.arff) + arff_tracecontent : List[List] + Arff representation (list of lists) of the trace data that was + generated by this fold (for putting in trace.arff) + user_defined_measures : Dict[float] + User defined measures that were generated on this fold + model : sklearn model + The model trained on this fold + """ + def _prediction_to_probabilities(y, model_classes): + # y: list or numpy array of predictions + # model_classes: sklearn classifier mapping from original array id to prediction index id + if not isinstance(model_classes, list): + raise ValueError('please convert model classes to list prior to calling this fn') + result = np.zeros((len(y), len(model_classes)), dtype=np.float32) + for obs, prediction_idx in enumerate(y): + array_idx = model_classes.index(prediction_idx) + result[obs][array_idx] = 1.0 + return result + + # TODO: if possible, give a warning if model is already fitted (acceptable in case of custom experimentation, + # but not desirable if we want to upload to OpenML). + + train_indices, test_indices = task.get_train_test_split_indices(repeat=rep_no, + fold=fold_no, + sample=sample_no) + + X, Y = task.get_X_and_y() + trainX = X[train_indices] + trainY = Y[train_indices] + testX = X[test_indices] + testY = Y[test_indices] + user_defined_measures = dict() + + try: + # for measuring runtime. Only available since Python 3.3 + if can_measure_runtime: + modelfit_starttime = time.process_time() + model.fit(trainX, trainY) + + if can_measure_runtime: + modelfit_duration = (time.process_time() - modelfit_starttime) * 1000 + user_defined_measures['usercpu_time_millis_training'] = modelfit_duration + except AttributeError as e: + # typically happens when training a regressor on classification task + raise PyOpenMLError(str(e)) + + # extract trace, if applicable + arff_tracecontent = [] + if isinstance(model, sklearn.model_selection._search.BaseSearchCV): + arff_tracecontent.extend(_extract_arfftrace(model, rep_no, fold_no)) + + # search for model classes_ (might differ depending on modeltype) + # first, pipelines are a special case (these don't have a classes_ + # object, but rather borrows it from the last step. We do this manually, + # because of the BaseSearch check) + if isinstance(model, sklearn.pipeline.Pipeline): + used_estimator = model.steps[-1][-1] + else: + used_estimator = model + + if isinstance(used_estimator, sklearn.model_selection._search.BaseSearchCV): + model_classes = used_estimator.best_estimator_.classes_ + else: + model_classes = used_estimator.classes_ + + if can_measure_runtime: + modelpredict_starttime = time.process_time() + + PredY = model.predict(testX) + try: + ProbaY = model.predict_proba(testX) + except AttributeError: + ProbaY = _prediction_to_probabilities(PredY, list(model_classes)) + + if can_measure_runtime: + modelpredict_duration = (time.process_time() - modelpredict_starttime) * 1000 + user_defined_measures['usercpu_time_millis_testing'] = modelpredict_duration + user_defined_measures['usercpu_time_millis'] = modelfit_duration + modelpredict_duration + + if ProbaY.shape[1] != len(task.class_labels): + warnings.warn("Repeat %d Fold %d: estimator only predicted for %d/%d classes!" % (rep_no, fold_no, ProbaY.shape[1], len(task.class_labels))) + + # add client-side calculated metrics. These might be used on the server as consistency check + def _calculate_local_measure(sklearn_fn, openml_name): + user_defined_measures[openml_name] = sklearn_fn(testY, PredY) + + _calculate_local_measure(sklearn.metrics.accuracy_score, 'predictive_accuracy') + + arff_datacontent = [] + for i in range(0, len(test_indices)): + arff_line = _prediction_to_row(rep_no, fold_no, sample_no, + test_indices[i], task.class_labels[testY[i]], + PredY[i], ProbaY[i], task.class_labels, model_classes) + arff_datacontent.append(arff_line) + return arff_datacontent, arff_tracecontent, user_defined_measures, model def _extract_arfftrace(model, rep_no, fold_no): @@ -526,11 +599,16 @@ def _extract_arfftrace_attributes(model): for key in model.cv_results_: if key.startswith('param_'): # supported types should include all types, including bool, int float - supported_types = (bool, int, float, six.string_types) - if all(isinstance(i, supported_types) or i is None for i in model.cv_results_[key]): - type = 'STRING' - else: - raise TypeError('Unsupported param type in param grid') + supported_basic_types = (bool, int, float, six.string_types) + for param_value in model.cv_results_[key]: + if isinstance(param_value, supported_basic_types) or param_value is None: + # basic string values + type = 'STRING' + elif isinstance(param_value, list) and all(isinstance(i, int) for i in param_value): + # list of integers + type = 'STRING' + else: + raise TypeError('Unsupported param type in param grid: %s' %key) # we renamed the attribute param to parameter, as this is a required # OpenML convention @@ -570,14 +648,17 @@ def get_run(run_id): run : OpenMLRun Run corresponding to ID, fetched from the server. """ - run_file = os.path.join(config.get_cache_directory(), "runs", - "run_%d.xml" % run_id) + run_dir = openml.utils._create_cache_directory_for_id(RUNS_CACHE_DIR_NAME, run_id) + run_file = os.path.join(run_dir, "description.xml") + + if not os.path.exists(run_dir): + os.makedirs(run_dir) try: return _get_cached_run(run_id) except (OpenMLCacheException): - run_xml = _perform_api_call("run/%d" % run_id) + run_xml = openml._api_calls._perform_api_call("run/%d" % run_id) with io.open(run_file, "w", encoding='utf8') as fh: fh.write(run_xml) @@ -586,7 +667,7 @@ def get_run(run_id): return run -def _create_run_from_xml(xml): +def _create_run_from_xml(xml, from_server=True): """Create a run object from xml returned from server. Parameters @@ -599,21 +680,37 @@ def _create_run_from_xml(xml): run : OpenMLRun New run object representing run_xml. """ - run = xmltodict.parse(xml)["oml:run"] - run_id = int(run['oml:run_id']) - uploader = int(run['oml:uploader']) - uploader_name = run['oml:uploader_name'] + + def obtain_field(xml_obj, fieldname, from_server, cast=None): + # this function can be used to check whether a field is present in an object. + # if it is not present, either returns None or throws an error (this is + # usually done if the xml comes from the server) + if fieldname in xml_obj: + if cast is not None: + return cast(xml_obj[fieldname]) + return xml_obj[fieldname] + elif not from_server: + return None + else: + raise AttributeError('Run XML does not contain required (server) field: ', fieldname) + + run = xmltodict.parse(xml, force_list=['oml:file', 'oml:evaluation'])["oml:run"] + run_id = obtain_field(run, 'oml:run_id', from_server, cast=int) + uploader = obtain_field(run, 'oml:uploader', from_server, cast=int) + uploader_name = obtain_field(run, 'oml:uploader_name', from_server) task_id = int(run['oml:task_id']) - task_type = run['oml:task_type'] + task_type = obtain_field(run, 'oml:task_type', from_server) + + # even with the server requirement this field may be empty. if 'oml:task_evaluation_measure' in run: task_evaluation_measure = run['oml:task_evaluation_measure'] else: task_evaluation_measure = None flow_id = int(run['oml:flow_id']) - flow_name = run['oml:flow_name'] - setup_id = int(run['oml:setup_id']) - setup_string = run['oml:setup_string'] + flow_name = obtain_field(run, 'oml:flow_name', from_server) + setup_id = obtain_field(run, 'oml:setup_id', from_server, cast=int) + setup_string = obtain_field(run, 'oml:setup_string', from_server) parameters = dict() if 'oml:parameter_settings' in run: @@ -623,7 +720,10 @@ def _create_run_from_xml(xml): value = parameter_dict['oml:value'] parameters[key] = value - dataset_id = int(run['oml:input_data']['oml:dataset']['oml:did']) + if 'oml:input_data' in run: + dataset_id = int(run['oml:input_data']['oml:dataset']['oml:did']) + elif not from_server: + dataset_id = None files = dict() evaluations = dict() @@ -632,21 +732,15 @@ def _create_run_from_xml(xml): if 'oml:output_data' not in run: raise ValueError('Run does not contain output_data (OpenML server error?)') else: - if isinstance(run['oml:output_data']['oml:file'], dict): - # only one result.. probably due to an upload error - file_dict = run['oml:output_data']['oml:file'] - files[file_dict['oml:name']] = int(file_dict['oml:file_id']) - elif isinstance(run['oml:output_data']['oml:file'], list): + output_data = run['oml:output_data'] + if 'oml:file' in output_data: # multiple files, the normal case - for file_dict in run['oml:output_data']['oml:file']: - files[file_dict['oml:name']] = int(file_dict['oml:file_id']) - else: - raise TypeError(type(run['oml:output_data']['oml:file'])) - - if 'oml:evaluation' in run['oml:output_data']: + for file_dict in output_data['oml:file']: + files[file_dict['oml:name']] = int(file_dict['oml:file_id']) + if 'oml:evaluation' in output_data: # in normal cases there should be evaluations, but in case there # was an error these could be absent - for evaluation_dict in run['oml:output_data']['oml:evaluation']: + for evaluation_dict in output_data['oml:evaluation']: key = evaluation_dict['oml:name'] if 'oml:value' in evaluation_dict: value = float(evaluation_dict['oml:value']) @@ -672,11 +766,11 @@ def _create_run_from_xml(xml): else: evaluations[key] = value - if 'description' not in files: + if 'description' not in files and from_server is True: raise ValueError('No description file for run %d in run ' 'description XML' % run_id) - if 'predictions' not in files: + if 'predictions' not in files and from_server is True: task = openml.tasks.get_task(task_id) if task.task_type_id == 8: raise NotImplementedError( @@ -789,11 +883,11 @@ def _create_trace_from_arff(arff_obj): def _get_cached_run(run_id): """Load a run from the cache.""" - cache_dir = config.get_cache_directory() - run_cache_dir = os.path.join(cache_dir, "runs") + run_cache_dir = openml.utils._create_cache_directory_for_id( + RUNS_CACHE_DIR_NAME, run_id, + ) try: - run_file = os.path.join(run_cache_dir, - "run_%d.xml" % int(run_id)) + run_file = os.path.join(run_cache_dir, "description.xml") with io.open(run_file, encoding='utf8') as fh: run = _create_run_from_xml(xml=fh.read()) return run @@ -804,10 +898,11 @@ def _get_cached_run(run_id): def list_runs(offset=None, size=None, id=None, task=None, setup=None, - flow=None, uploader=None, tag=None, display_errors=False): - """List all runs matching all of the given filters. + flow=None, uploader=None, tag=None, display_errors=False, **kwargs): - Perform API call `/run/list/{filters} `_ + """ + List all runs matching all of the given filters. + (Supports large amount of results) Parameters ---------- @@ -831,17 +926,61 @@ def list_runs(offset=None, size=None, id=None, task=None, setup=None, display_errors : bool, optional (default=None) Whether to list runs which have an error (for example a missing prediction file). + + kwargs: dict, optional + Legal filter operators: task_type. + Returns ------- - list + dict + List of found runs. + """ + + return openml.utils.list_all(_list_runs, offset=offset, size=size, id=id, task=task, setup=setup, + flow=flow, uploader=uploader, tag=tag, display_errors=display_errors, **kwargs) + + +def _list_runs(id=None, task=None, setup=None, + flow=None, uploader=None, display_errors=False, **kwargs): + + """ + Perform API call `/run/list/{filters}' + ` + + Parameters + ---------- + The arguments that are lists are separated from the single value + ones which are put into the kwargs. + display_errors is also separated from the kwargs since it has a + default value. + + id : list, optional + + task : list, optional + + setup: list, optional + + flow : list, optional + + uploader : list, optional + + display_errors : bool, optional (default=None) + Whether to list runs which have an error (for example a missing + prediction file). + + kwargs: dict, optional + Legal filter operators: task_type. + + Returns + ------- + dict List of found runs. """ api_call = "run/list" - if offset is not None: - api_call += "/offset/%d" % int(offset) - if size is not None: - api_call += "/limit/%d" % int(size) + if kwargs is not None: + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) if id is not None: api_call += "/run/%s" % ','.join([str(int(i)) for i in id]) if task is not None: @@ -852,19 +991,14 @@ def list_runs(offset=None, size=None, id=None, task=None, setup=None, api_call += "/flow/%s" % ','.join([str(int(i)) for i in flow]) if uploader is not None: api_call += "/uploader/%s" % ','.join([str(int(i)) for i in uploader]) - if tag is not None: - api_call += "/tag/%s" % tag if display_errors: api_call += "/show_errors/true" + return __list_runs(api_call) - return _list_runs(api_call) - -def _list_runs(api_call): +def __list_runs(api_call): """Helper function to parse API calls which are lists of runs""" - - xml_string = _perform_api_call(api_call) - + xml_string = openml._api_calls._perform_api_call(api_call) runs_dict = xmltodict.parse(xml_string, force_list=('oml:run',)) # Minimalistic check if the XML is useful if 'oml:runs' not in runs_dict: diff --git a/openml/runs/run.py b/openml/runs/run.py index 567533679..9d80999d6 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -1,4 +1,4 @@ -from collections import OrderedDict, defaultdict +from collections import OrderedDict import json import sys import time @@ -8,10 +8,11 @@ import xmltodict import openml +import openml._api_calls from ..tasks import get_task -from .._api_calls import _perform_api_call, _file_id_to_url, _read_url_files from ..exceptions import PyOpenMLError + class OpenMLRun(object): """OpenML Run: result of running a model on an openml dataset. @@ -53,6 +54,17 @@ def __init__(self, task_id, flow_id, dataset_id, setup_string=None, self.tags = tags self.predictions_url = predictions_url + def __str__(self): + flow_name = self.flow_name + if len(flow_name) > 26: + # long enough to show sklearn.pipeline.Pipeline + flow_name = flow_name[:26] + "..." + return "[run id: {}, task id: {}, flow id: {}, flow name: {}]".format( + self.run_id, self.task_id, self.flow_id, flow_name) + + def _repr_pretty_(self, pp, cycle): + pp.text(str(self)) + def _generate_arff_dict(self): """Generates the arff dictionary for uploading predictions to the server. @@ -109,27 +121,29 @@ def _generate_trace_arff_dict(self): return arff_dict def get_metric_fn(self, sklearn_fn, kwargs={}): - '''Calculates metric scores based on predicted values. Assumes the + """Calculates metric scores based on predicted values. Assumes the run has been executed locally (and contains run_data). Furthermore, it assumes that the 'correct' attribute is specified in the arff (which is an optional field, but always the case for openml-python runs) Parameters - ------- + ---------- sklearn_fn : function a function pointer to a sklearn function that - accepts y_true, y_pred and *kwargs + accepts ``y_true``, ``y_pred`` and ``**kwargs`` Returns ------- scores : list a list of floats, of length num_folds * num_repeats - ''' + """ if self.data_content is not None and self.task_id is not None: predictions_arff = self._generate_arff_dict() elif 'predictions' in self.output_files: - predictions_file_url = _file_id_to_url(self.output_files['predictions'], 'predictions.arff') + predictions_file_url = openml._api_calls._file_id_to_url( + self.output_files['predictions'], 'predictions.arff', + ) predictions_arff = arff.loads(openml._api_calls._read_url(predictions_file_url)) # TODO: make this a stream reader else: @@ -222,7 +236,7 @@ def publish(self): trace_arff = arff.dumps(self._generate_trace_arff_dict()) file_elements['trace'] = ("trace.arff", trace_arff) - return_value = _perform_api_call("/run/", file_elements=file_elements) + return_value = openml._api_calls._perform_api_call("/run/", file_elements=file_elements) run_id = int(xmltodict.parse(return_value)['oml:upload_run']['oml:run_id']) self.run_id = run_id return self @@ -349,6 +363,28 @@ def extract_parameters(_flow, _flow_dict, component_model, return parameters + def push_tag(self, tag): + """Annotates this run with a tag on the server. + + Parameters + ---------- + tag : str + Tag to attach to the run. + """ + data = {'run_id': self.run_id, 'tag': tag} + openml._api_calls._perform_api_call("/run/tag", data=data) + + def remove_tag(self, tag): + """Removes a tag from this run on the server. + + Parameters + ---------- + tag : str + Tag to attach to the run. + """ + data = {'run_id': self.run_id, 'tag': tag} + openml._api_calls._perform_api_call("/run/untag", data=data) + ################################################################################ # Functions which cannot be in runs/functions due to circular imports diff --git a/openml/setups/functions.py b/openml/setups/functions.py index c5c2652a7..745da5a1e 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -1,18 +1,23 @@ from collections import OrderedDict +import io import openml +import os import xmltodict +from .. import config from .setup import OpenMLSetup, OpenMLParameter from openml.flows import flow_exists +from openml.exceptions import OpenMLServerNoResult +import openml.utils def setup_exists(flow, model=None): ''' Checks whether a hyperparameter configuration already exists on the server. - Parameter - --------- + Parameters + ---------- flow : flow The openml flow object. @@ -54,8 +59,23 @@ def setup_exists(flow, model=None): return False +def _get_cached_setup(setup_id): + """Load a run from the cache.""" + cache_dir = config.get_cache_directory() + setup_cache_dir = os.path.join(cache_dir, "setups", str(setup_id)) + try: + setup_file = os.path.join(setup_cache_dir, "description.xml") + with io.open(setup_file, encoding='utf8') as fh: + setup_xml = xmltodict.parse(fh.read()) + setup = _create_setup_from_xml(setup_xml) + return setup + + except (OSError, IOError): + raise openml.exceptions.OpenMLCacheException("Setup file for setup id %d not cached" % setup_id) + + def get_setup(setup_id): - ''' + """ Downloads the setup (configuration) description from OpenML and returns a structured object @@ -68,54 +88,78 @@ def get_setup(setup_id): ------- OpenMLSetup an initialized openml setup object - ''' - result = openml._api_calls._perform_api_call('/setup/%d' %setup_id) - result_dict = xmltodict.parse(result) + """ + setup_dir = os.path.join(config.get_cache_directory(), "setups", str(setup_id)) + setup_file = os.path.join(setup_dir, "description.xml") + + if not os.path.exists(setup_dir): + os.makedirs(setup_dir) + + try: + return _get_cached_setup(setup_id) + + except (openml.exceptions.OpenMLCacheException): + setup_xml = openml._api_calls._perform_api_call('/setup/%d' % setup_id) + with io.open(setup_file, "w", encoding='utf8') as fh: + fh.write(setup_xml) + + result_dict = xmltodict.parse(setup_xml) return _create_setup_from_xml(result_dict) -def list_setups(flow=None, tag=None, setup=None, offset=None, size=None): - """List all setups matching all of the given filters. +def list_setups(offset=None, size=None, flow=None, tag=None, setup=None): + """ + List all setups matching all of the given filters. - Perform API call `/setup/list/{filters} + Parameters + ---------- + offset : int, optional + size : int, optional + flow : int, optional + tag : str, optional + setup : list(int), optional - Parameters - ---------- - flow : int, optional + Returns + ------- + dict + """ - tag : str, optional + return openml.utils.list_all(_list_setups, offset=offset, size=size, + flow=flow, tag=tag, setup=setup) - setup : list(int), optional - offset : int, optional +def _list_setups(setup=None, **kwargs): + """ + Perform API call `/setup/list/{filters}` - size : int, optional + Parameters + ---------- + The setup argument that is a list is separated from the single value + filters which are put into the kwargs. - Returns - ------- - dict + setup : list(int), optional + + kwargs: dict, optional + Legal filter operators: flow, setup, limit, offset, tag. + + Returns + ------- + dict """ api_call = "setup/list" - if offset is not None: - api_call += "/offset/%d" % int(offset) - if size is not None: - api_call += "/limit/%d" % int(size) if setup is not None: api_call += "/setup/%s" % ','.join([str(int(i)) for i in setup]) - if flow is not None: - api_call += "/flow/%s" % flow - if tag is not None: - api_call += "/tag/%s" % tag + if kwargs is not None: + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) - return _list_setups(api_call) + return __list_setups(api_call) -def _list_setups(api_call): +def __list_setups(api_call): """Helper function to parse API calls which are lists of setups""" - xml_string = openml._api_calls._perform_api_call(api_call) - setups_dict = xmltodict.parse(xml_string, force_list=('oml:setup',)) # Minimalistic check if the XML is useful if 'oml:setups' not in setups_dict: @@ -212,6 +256,7 @@ def _to_dict(flow_id, openml_parameter_settings): return xml + def _create_setup_from_xml(result_dict): ''' Turns an API xml result into a OpenMLSetup object diff --git a/openml/study/functions.py b/openml/study/functions.py index 11c47d674..cce4ca4b0 100644 --- a/openml/study/functions.py +++ b/openml/study/functions.py @@ -1,7 +1,8 @@ import xmltodict from openml.study import OpenMLStudy -from .._api_calls import _perform_api_call +import openml._api_calls + def _multitag_to_list(result_dict, tag): if isinstance(result_dict[tag], list): @@ -21,7 +22,7 @@ def get_study(study_id, type=None): call_suffix = "study/%s" %str(study_id) if type is not None: call_suffix += "/" + type - xml_string = _perform_api_call(call_suffix) + xml_string = openml._api_calls._perform_api_call(call_suffix) result_dict = xmltodict.parse(xml_string)['oml:study'] id = int(result_dict['oml:id']) name = result_dict['oml:name'] @@ -55,4 +56,4 @@ def get_study(study_id, type=None): study = OpenMLStudy(id, name, description, creation_date, creator, tags, datasets, tasks, flows, setups) - return study \ No newline at end of file + return study diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 3654b8603..0fbdc9b21 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -2,26 +2,25 @@ import io import re import os -import shutil from oslo_concurrency import lockutils import xmltodict from ..exceptions import OpenMLCacheException from ..datasets import get_dataset -from .task import OpenMLTask, _create_task_cache_dir -from .. import config -from .._api_calls import _perform_api_call +from .task import OpenMLTask +import openml.utils +import openml._api_calls + +TASKS_CACHE_DIR_NAME = 'tasks' def _get_cached_tasks(): tasks = OrderedDict() - cache_dir = config.get_cache_directory() - task_cache_dir = os.path.join(cache_dir, "tasks") + task_cache_dir = openml.utils._create_cache_directory(TASKS_CACHE_DIR_NAME) directory_content = os.listdir(task_cache_dir) directory_content.sort() - # Find all dataset ids for which we have downloaded the dataset # description @@ -36,15 +35,19 @@ def _get_cached_tasks(): def _get_cached_task(tid): - cache_dir = config.get_cache_directory() - task_cache_dir = os.path.join(cache_dir, "tasks") - task_file = os.path.join(task_cache_dir, str(tid), "task.xml") + + tid_cache_dir = openml.utils._create_cache_directory_for_id( + TASKS_CACHE_DIR_NAME, + tid + ) + task_file = os.path.join(tid_cache_dir, "task.xml") try: with io.open(task_file, encoding='utf8') as fh: task = _create_task_from_xml(xml=fh.read()) return task except (OSError, IOError): + openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir) raise OpenMLCacheException("Task file for tid %d not " "cached" % tid) @@ -55,12 +58,12 @@ def _get_estimation_procedure_list(): Returns ------- procedures : list - A list of all estimation procedures. Every procedure is represented by a - dictionary containing the following information: id, - task type id, name, type, repeats, folds, stratified. + A list of all estimation procedures. Every procedure is represented by + a dictionary containing the following information: id, task type id, + name, type, repeats, folds, stratified. """ - xml_string = _perform_api_call("estimationprocedure/list") + xml_string = openml._api_calls._perform_api_call("estimationprocedure/list") procs_dict = xmltodict.parse(xml_string) # Minimalistic check if the XML is useful if 'oml:estimationprocedures' not in procs_dict: @@ -88,14 +91,28 @@ def _get_estimation_procedure_list(): return procs -def list_tasks(task_type_id=None, offset=None, size=None, tag=None): - """Return a number of tasks having the given tag and task_type_id +def list_tasks(task_type_id=None, offset=None, size=None, tag=None, **kwargs): + """ + Return a number of tasks having the given tag and task_type_id Parameters ---------- + Filter task_type_id is separated from the other filters because + it is used as task_type_id in the task description, but it is named + type when used as a filter in list tasks call. + task_type_id : int, optional ID of the task type as detailed `here `_. + + - Supervised classification: 1 + - Supervised regression: 2 + - Learning curve: 3 + - Supervised data stream classification: 4 + - Clustering: 5 + - Machine Learning Challenge: 6 + - Survival Analysis: 7 + - Subgroup Discovery: 8 offset : int, optional the number of tasks to skip, starting from the first size : int, optional @@ -103,6 +120,10 @@ def list_tasks(task_type_id=None, offset=None, size=None, tag=None): tag : str, optional the tag to include + kwargs: dict, optional + Legal filter operators: data_tag, status, data_id, data_name, number_instances, number_features, + number_classes, number_missing_values. + Returns ------- dict @@ -111,25 +132,54 @@ def list_tasks(task_type_id=None, offset=None, size=None, tag=None): task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned. """ - api_call = "task/list" - if task_type_id is not None: - api_call += "/type/%d" % int(task_type_id) + return openml.utils.list_all(_list_tasks, task_type_id=task_type_id, offset=offset, size=size, tag=tag, **kwargs) + + +def _list_tasks(task_type_id=None, **kwargs): + """ + Perform the api call to return a number of tasks having the given filters. - if offset is not None: - api_call += "/offset/%d" % int(offset) + Parameters + ---------- + Filter task_type_id is separated from the other filters because + it is used as task_type_id in the task description, but it is named + type when used as a filter in list tasks call. - if size is not None: - api_call += "/limit/%d" % int(size) + task_type_id : int, optional + ID of the task type as detailed + `here `_. - if tag is not None: - api_call += "/tag/%s" % tag + - Supervised classification: 1 + - Supervised regression: 2 + - Learning curve: 3 + - Supervised data stream classification: 4 + - Clustering: 5 + - Machine Learning Challenge: 6 + - Survival Analysis: 7 + - Subgroup Discovery: 8 - return _list_tasks(api_call) + kwargs: dict, optional + Legal filter operators: tag, data_tag, status, limit, + offset, data_id, data_name, number_instances, number_features, + number_classes, number_missing_values. + Returns + ------- + dict + """ + api_call = "task/list" + if task_type_id is not None: + api_call += "/type/%d" % int(task_type_id) + if kwargs is not None: + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) + return __list_tasks(api_call) -def _list_tasks(api_call): - xml_string = _perform_api_call(api_call) - tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task',)) + +def __list_tasks(api_call): + + xml_string = openml._api_calls._perform_api_call(api_call) + tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task', 'oml:input')) # Minimalistic check if the XML is useful if 'oml:tasks' not in tasks_dict: raise ValueError('Error in return XML, does not contain "oml:runs": %s' @@ -228,11 +278,13 @@ def get_task(task_id): raise ValueError("Task ID is neither an Integer nor can be " "cast to an Integer.") - tid_cache_dir = _create_task_cache_dir(task_id) + tid_cache_dir = openml.utils._create_cache_directory_for_id( + TASKS_CACHE_DIR_NAME, task_id, + ) with lockutils.external_lock( - name='datasets.functions.get_dataset:%d' % task_id, - lock_path=os.path.join(config.get_cache_directory(), 'locks'), + name='task.functions.get_task:%d' % task_id, + lock_path=openml.utils._create_lockfiles_dir(), ): try: task = _get_task_description(task_id) @@ -240,9 +292,8 @@ def get_task(task_id): class_labels = dataset.retrieve_class_labels(task.target_name) task.class_labels = class_labels task.download_split() - except Exception as e: - _remove_task_cache_dir(tid_cache_dir) + openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir) raise e return task @@ -253,8 +304,11 @@ def _get_task_description(task_id): try: return _get_cached_task(task_id) except OpenMLCacheException: - xml_file = os.path.join(_create_task_cache_dir(task_id), "task.xml") - task_xml = _perform_api_call("task/%d" % task_id) + xml_file = os.path.join( + openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id), + "task.xml", + ) + task_xml = openml._api_calls._perform_api_call("task/%d" % task_id) with io.open(xml_file, "w", encoding='utf8') as fh: fh.write(task_xml) @@ -263,53 +317,6 @@ def _get_task_description(task_id): return task -def _create_task_cache_directory(task_id): - """Create a task cache directory - - In order to have a clearer cache structure and because every task - is cached in several files (description, split), there - is a directory for each task witch the task ID being the directory - name. This function creates this cache directory. - - This function is NOT thread/multiprocessing safe. - - Parameters - ---------- - tid : int - Task ID - - Returns - ------- - str - Path of the created dataset cache directory. - """ - task_cache_dir = os.path.join( - config.get_cache_directory(), "tasks", str(task_id) - ) - if os.path.exists(task_cache_dir) and os.path.isdir(task_cache_dir): - pass - elif os.path.exists(task_cache_dir) and not os.path.isdir(task_cache_dir): - raise ValueError('Task cache dir exists but is not a directory!') - else: - os.makedirs(task_cache_dir) - return task_cache_dir - - -def _remove_task_cache_dir(tid_cache_dir): - """Remove the task cache directory - - This function is NOT thread/multiprocessing safe. - - Parameters - ---------- - """ - try: - shutil.rmtree(tid_cache_dir) - except (OSError, IOError): - raise ValueError('Cannot remove faulty task cache directory %s.' - 'Please do this manually!' % tid_cache_dir) - - def _create_task_from_xml(xml): dic = xmltodict.parse(xml)["oml:task"] diff --git a/openml/tasks/split.py b/openml/tasks/split.py index 6b7c7d0eb..6f4b13730 100644 --- a/openml/tasks/split.py +++ b/openml/tasks/split.py @@ -1,6 +1,6 @@ from collections import namedtuple, OrderedDict import os -import sys +import six import numpy as np import scipy.io.arff @@ -10,6 +10,10 @@ Split = namedtuple("Split", ["train", "test"]) +if six.PY2: + FileNotFoundError = IOError + + class OpenMLSplit(object): def __init__(self, name, description, split): @@ -60,17 +64,26 @@ def __eq__(self, other): @classmethod def _from_arff_file(cls, filename, cache=True): repetitions = None - pkl_filename = filename.replace(".arff", ".pkl") + if six.PY2: + pkl_filename = filename.replace(".arff", ".pkl.py2") + else: + pkl_filename = filename.replace(".arff", ".pkl.py3") if cache: if os.path.exists(pkl_filename): - with open(pkl_filename, "rb") as fh: - _ = pickle.load(fh) + try: + with open(pkl_filename, "rb") as fh: + _ = pickle.load(fh) + except UnicodeDecodeError as e: + # Possibly pickle file was created with python2 and python3 is being used to load the data + raise e repetitions = _["repetitions"] name = _["name"] # Cache miss if repetitions is None: # Faster than liac-arff and sufficient in this situation! + if not os.path.exists(filename): + raise FileNotFoundError('Split arff %s does not exist!' % filename) splits, meta = scipy.io.arff.loadarff(filename) name = meta.name diff --git a/openml/tasks/task.py b/openml/tasks/task.py index 127e7e232..cc7dd6731 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -4,7 +4,8 @@ from .. import config from .. import datasets from .split import OpenMLSplit -from .._api_calls import _read_url +import openml._api_calls +from ..utils import _create_cache_directory_for_id class OpenMLTask(object): @@ -36,21 +37,17 @@ def get_dataset(self): return datasets.get_dataset(self.dataset_id) def get_X_and_y(self): + """Get data associated with the current task. + + Returns + ------- + tuple - X and y + + """ dataset = self.get_dataset() - # Replace with retrieve from cache - if self.task_type_id == 1: - # if 'Supervised Classification'.lower() in self.task_type.lower(): - target_dtype = int - # elif 'Supervised Regression'.lower() in self.task_type.lower(): - elif self.task_type_id == 2: - target_dtype = float - # elif ''.lower('Learning Curve') in self.task_type.lower(): - elif self.task_type_id == 3: - target_dtype = int - else: + if self.task_type_id not in (1, 2, 3): raise NotImplementedError(self.task_type) - X_and_y = dataset.get_data(target=self.target_name, - target_dtype=target_dtype) + X_and_y = dataset.get_data(target=self.target_name) return X_and_y def get_train_test_split_indices(self, fold=0, repeat=0, sample=0): @@ -67,7 +64,7 @@ def _download_split(self, cache_file): pass except (OSError, IOError): split_url = self.estimation_procedure["data_splits_url"] - split_arff = _read_url(split_url) + split_arff = openml._api_calls._read_url(split_url) with io.open(cache_file, "w", encoding='utf8') as fh: fh.write(split_arff) @@ -77,12 +74,12 @@ def download_split(self): """Download the OpenML split for a given task. """ cached_split_file = os.path.join( - _create_task_cache_dir(self.task_id), "datasplits.arff") + _create_cache_directory_for_id('tasks', self.task_id), + "datasplits.arff", + ) try: split = OpenMLSplit._from_arff_file(cached_split_file) - # Add FileNotFoundError in python3 version (which should be a - # subclass of OSError. except (OSError, IOError): # Next, download and cache the associated split file self._download_split(cached_split_file) @@ -96,6 +93,28 @@ def get_split_dimensions(self): return self.split.repeats, self.split.folds, self.split.samples + def push_tag(self, tag): + """Annotates this task with a tag on the server. + + Parameters + ---------- + tag : str + Tag to attach to the task. + """ + data = {'task_id': self.task_id, 'tag': tag} + openml._api_calls._perform_api_call("/task/tag", data=data) + + def remove_tag(self, tag): + """Removes a tag from this task on the server. + + Parameters + ---------- + tag : str + Tag to attach to the task. + """ + data = {'task_id': self.task_id, 'tag': tag} + openml._api_calls._perform_api_call("/task/untag", data=data) + def _create_task_cache_dir(task_id): task_cache_dir = os.path.join(config.get_cache_directory(), "tasks", str(task_id)) diff --git a/openml/testing.py b/openml/testing.py index 62c383a95..0b75da06f 100644 --- a/openml/testing.py +++ b/openml/testing.py @@ -26,7 +26,6 @@ def setUp(self): self.maxDiff = None self.static_cache_dir = None static_cache_dir = os.path.dirname(os.path.abspath(inspect.getfile(self.__class__))) - static_cache_dir = os.path.abspath(os.path.join(static_cache_dir, '..')) content = os.listdir(static_cache_dir) if 'files' in content: @@ -52,10 +51,12 @@ def setUp(self): openml.config.apikey = "610344db6388d9ba34f6db45a3cf71de" self.production_server = openml.config.server self.test_server = "https://test.openml.org/api/v1/xml" + openml.config.cache_directory = None + openml.config.server = self.test_server openml.config.avoid_duplicate_runs = False - openml.config.set_cache_directory(self.workdir) + openml.config.cache_directory = self.workdir # If we're on travis, we save the api key in the config file to allow # the notebook tests to read them. diff --git a/openml/utils.py b/openml/utils.py index 7eb4c465a..afe83f141 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -1,5 +1,10 @@ +import os +import xmltodict import six +import shutil +import openml._api_calls +from . import config from openml.exceptions import OpenMLServerException @@ -40,8 +45,56 @@ def extract_xml_tags(xml_tag_name, node, allow_none=True): else: raise ValueError("Could not find tag '%s' in node '%s'" % (xml_tag_name, str(node))) - -def list_all(listing_call, batch_size=10000, *args, **filters): + +def _tag_entity(entity_type, entity_id, tag, untag=False): + """Function that tags or untags a given entity on OpenML. As the OpenML + API tag functions all consist of the same format, this function covers + all entity types (currently: dataset, task, flow, setup, run). Could + be used in a partial to provide dataset_tag, dataset_untag, etc. + + Parameters + ---------- + entity_type : str + Name of the entity to tag (e.g., run, flow, data) + + entity_id : int + OpenML id of the entity + + tag : str + The tag + + untag : bool + Set to true if needed to untag, rather than tag + + Returns + ------- + tags : list + List of tags that the entity is (still) tagged with + """ + legal_entities = {'data', 'task', 'flow', 'setup', 'run'} + if entity_type not in legal_entities: + raise ValueError('Can\'t tag a %s' %entity_type) + + uri = '%s/tag' %entity_type + main_tag = 'oml:%s_tag' %entity_type + if untag: + uri = '%s/untag' %entity_type + main_tag = 'oml:%s_untag' %entity_type + + + post_variables = {'%s_id'%entity_type: entity_id, 'tag': tag} + result_xml = openml._api_calls._perform_api_call(uri, post_variables) + + result = xmltodict.parse(result_xml, force_list={'oml:tag'})[main_tag] + + if 'oml:tag' in result: + return result['oml:tag'] + else: + # no tags, return empty list + return [] + + +def list_all(listing_call, *args, **filters): """Helper to handle paged listing requests. Example usage: @@ -55,8 +108,6 @@ def list_all(listing_call, batch_size=10000, *args, **filters): ---------- listing_call : callable Call listing, e.g. list_evaluations. - batch_size : int (default: 10000) - Batch size for paging. *args : Variable length argument list Any required arguments for the listing call. **filters : Arbitrary keyword arguments @@ -66,16 +117,34 @@ def list_all(listing_call, batch_size=10000, *args, **filters): ------- dict """ + + # default batch size per paging. + batch_size = 10000 + # eliminate filters that have a None value + active_filters = {key: value for key, value in filters.items() if value is not None} page = 0 result = {} - - while True: + # max number of results to be shown + limit = None + offset = 0 + cycle = True + if 'size' in active_filters: + limit = active_filters['size'] + del active_filters['size'] + # check if the batch size is greater than the number of results that need to be returned. + if limit is not None: + if batch_size > limit: + batch_size = limit + if 'offset' in active_filters: + offset = active_filters['offset'] + del active_filters['offset'] + while cycle: try: new_batch = listing_call( *args, - size=batch_size, - offset=batch_size*page, - **filters + limit=batch_size, + offset=offset + batch_size * page, + **active_filters ) except OpenMLServerException as e: if page == 0 and e.args[0] == 'No results': @@ -84,5 +153,83 @@ def list_all(listing_call, batch_size=10000, *args, **filters): break result.update(new_batch) page += 1 + if limit is not None: + limit -= batch_size + # check if the number of required results has been achieved + if limit == 0: + break + # check if there are enough results to fulfill a batch + if limit < batch_size: + batch_size = limit return result + + +def _create_cache_directory(key): + cache = config.get_cache_directory() + cache_dir = os.path.join(cache, key) + try: + os.makedirs(cache_dir) + except: + pass + return cache_dir + + +def _create_cache_directory_for_id(key, id_): + """Create the cache directory for a specific ID + + In order to have a clearer cache structure and because every task + is cached in several files (description, split), there + is a directory for each task witch the task ID being the directory + name. This function creates this cache directory. + + This function is NOT thread/multiprocessing safe. + + Parameters + ---------- + key : str + + id_ : int + + Returns + ------- + str + Path of the created dataset cache directory. + """ + cache_dir = os.path.join( + _create_cache_directory(key), str(id_) + ) + if os.path.exists(cache_dir) and os.path.isdir(cache_dir): + pass + elif os.path.exists(cache_dir) and not os.path.isdir(cache_dir): + raise ValueError('%s cache dir exists but is not a directory!' % key) + else: + os.makedirs(cache_dir) + return cache_dir + + +def _remove_cache_dir_for_id(key, cache_dir): + """Remove the task cache directory + + This function is NOT thread/multiprocessing safe. + + Parameters + ---------- + key : str + + cache_dir : str + """ + try: + shutil.rmtree(cache_dir) + except (OSError, IOError): + raise ValueError('Cannot remove faulty %s cache directory %s.' + 'Please do this manually!' % (key, cache_dir)) + + +def _create_lockfiles_dir(): + dir = os.path.join(config.get_cache_directory(), 'locks') + try: + os.makedirs(dir) + except: + pass + return dir diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e5aa16739..000000000 --- a/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -mock -numpy>=1.6.2 -scipy>=0.13.3 -liac-arff>=2.1.1 -xmltodict -nose -requests -scikit-learn>=0.18 -nbformat -python-dateutil -oslo.concurrency \ No newline at end of file diff --git a/setup.py b/setup.py index f9cfeefa1..a0cfb6e66 100644 --- a/setup.py +++ b/setup.py @@ -1,27 +1,12 @@ # -*- coding: utf-8 -*- -import os import setuptools import sys with open("openml/__version__.py") as fh: version = fh.readlines()[-1].split()[-1].strip("\"'") - -requirements_file = os.path.join(os.path.dirname(__file__), 'requirements.txt') -requirements = [] dependency_links = [] -with open(requirements_file) as fh: - for line in fh: - line = line.strip() - if line: - # Make sure the github URLs work here as well - split = line.split('@') - split = split[0] - split = split.split('/') - url = '/'.join(split[:-1]) - requirement = split[-1] - requirements.append(requirement) try: import numpy @@ -43,16 +28,34 @@ maintainer="Matthias Feurer", maintainer_email="feurerm@informatik.uni-freiburg.de", description="Python API for OpenML", - license="GPLv3", + license="BSD 3-clause", url="http://openml.org/", version=version, packages=setuptools.find_packages(), package_data={'': ['*.txt', '*.md']}, - install_requires=requirements, + install_requires=[ + 'mock', + 'numpy>=1.6.2', + 'scipy>=0.13.3', + 'liac-arff>=2.2.1', + 'xmltodict', + 'nose', + 'requests', + 'scikit-learn>=0.18', + 'nbformat', + 'python-dateutil', + 'oslo.concurrency', + ], + extras_require={ + 'test': [ + 'nbconvert', + 'jupyter_client' + ] + }, test_suite="nose.collector", classifiers=['Intended Audience :: Science/Research', 'Intended Audience :: Developers', - 'License :: GPLv3', + 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Topic :: Software Development', 'Topic :: Scientific/Engineering', diff --git a/tests/files/datasets/-1/dataset.arff b/tests/files/org/openml/test/datasets/-1/dataset.arff similarity index 100% rename from tests/files/datasets/-1/dataset.arff rename to tests/files/org/openml/test/datasets/-1/dataset.arff diff --git a/tests/files/datasets/-1/description.xml b/tests/files/org/openml/test/datasets/-1/description.xml similarity index 100% rename from tests/files/datasets/-1/description.xml rename to tests/files/org/openml/test/datasets/-1/description.xml diff --git a/tests/files/datasets/-1/features.xml b/tests/files/org/openml/test/datasets/-1/features.xml similarity index 100% rename from tests/files/datasets/-1/features.xml rename to tests/files/org/openml/test/datasets/-1/features.xml diff --git a/tests/files/datasets/-1/qualities.xml b/tests/files/org/openml/test/datasets/-1/qualities.xml similarity index 100% rename from tests/files/datasets/-1/qualities.xml rename to tests/files/org/openml/test/datasets/-1/qualities.xml diff --git a/tests/files/datasets/2/dataset.arff b/tests/files/org/openml/test/datasets/2/dataset.arff similarity index 100% rename from tests/files/datasets/2/dataset.arff rename to tests/files/org/openml/test/datasets/2/dataset.arff diff --git a/tests/files/datasets/2/description.xml b/tests/files/org/openml/test/datasets/2/description.xml similarity index 100% rename from tests/files/datasets/2/description.xml rename to tests/files/org/openml/test/datasets/2/description.xml diff --git a/tests/files/datasets/2/features.xml b/tests/files/org/openml/test/datasets/2/features.xml similarity index 100% rename from tests/files/datasets/2/features.xml rename to tests/files/org/openml/test/datasets/2/features.xml diff --git a/tests/files/datasets/2/qualities.xml b/tests/files/org/openml/test/datasets/2/qualities.xml similarity index 100% rename from tests/files/datasets/2/qualities.xml rename to tests/files/org/openml/test/datasets/2/qualities.xml diff --git a/tests/files/org/openml/test/runs/1/description.xml b/tests/files/org/openml/test/runs/1/description.xml new file mode 100644 index 000000000..92e9bcb98 --- /dev/null +++ b/tests/files/org/openml/test/runs/1/description.xml @@ -0,0 +1,645 @@ + + 100 + 1 + Jan van Rijn + 28 + Supervised Classification + 67 + weka.BayesNet_K2(1) + 12 + weka.classifiers.bayes.BayesNet -- -D -Q weka.classifiers.bayes.net.search.local.K2 -- -P 1 -S BAYES -E weka.classifiers.bayes.net.estimate.SimpleEstimator -- -A 0.5 + + D + true + + + Q + weka.classifiers.bayes.net.search.local.K2 + + + P + 1 + + + S + BAYES + + + + 28 + optdigits + https://www.openml.org/data/download/28/dataset_28_optdigits.arff + + + + + -1 + 261 + description + https://www.openml.org/data/download/261/weka_generated_run935374685998857626.xml + + + -1 + 262 + predictions + https://www.openml.org/data/download/262/weka_generated_predictions576954524972002741.arff + + + area_under_roc_curve + 0.990288 [0.99724,0.989212,0.992776,0.994279,0.980578,0.98649,0.99422,0.99727,0.994858,0.976143] + + average_cost + 0 + + f_measure + 0.922723 [0.989091,0.898857,0.935041,0.92431,0.927944,0.918156,0.980322,0.933219,0.895018,0.826531] + + kappa + 0.913601 + + kb_relative_information_score + 5181.417432 + + mean_absolute_error + 0.016374 + + mean_prior_absolute_error + 0.179997 + + number_of_instances + 5620 [554,571,557,572,568,558,558,566,554,562] + + os_information + [ Oracle Corporation, 1.7.0_51, amd64, Linux, 3.7.10-1.28-desktop ] + + precision + 0.924345 [0.996337,0.902827,0.953358,0.941924,0.926316,0.966337,0.978571,0.905316,0.882456,0.791531] + + predictive_accuracy + 0.922242 + + prior_entropy + 3.321833 + + recall + 0.922242 [0.981949,0.894921,0.917415,0.907343,0.929577,0.874552,0.982079,0.962898,0.907942,0.864769] + + relative_absolute_error + 0.090968 + + root_mean_prior_squared_error + 0.299998 + + root_mean_squared_error + 0.117387 + + root_relative_squared_error + 0.391293 + + scimark_benchmark + 1969.9216824070186 [ 1241.8943613564243, 1575.5968355392279, 906.1111964820476, 1675.0415998938465, 4450.964418763549 ] + + total_cost + 0 + + area_under_roc_curve + 0.993338 [1,0.987789,0.9958,0.998255,0.988484,0.997318,0.989483,0.999259,0.99749,0.979708] + + area_under_roc_curve + 0.990543 [0.999928,0.977523,0.996012,0.995758,0.970974,0.978296,0.999929,0.998447,0.995804,0.993418] + + area_under_roc_curve + 0.990071 [0.990873,0.996839,0.994512,0.994268,0.988327,0.989448,0.999965,0.997881,0.994549,0.95384] + + area_under_roc_curve + 0.988339 [1,0.983081,0.979249,0.989456,0.989699,0.976937,1,0.999375,0.996199,0.969597] + + area_under_roc_curve + 0.993133 [1,0.992201,0.988848,1,0.998749,0.996288,0.981719,0.998506,0.996378,0.978826] + + area_under_roc_curve + 0.990209 [0.990837,0.997777,0.9982,0.997151,0.9786,0.985602,0.99086,0.990412,0.99125,0.981449] + + area_under_roc_curve + 0.989645 [0.990613,0.998558,0.999014,0.985044,0.980059,0.986784,0.990335,0.996161,0.998094,0.971944] + + area_under_roc_curve + 0.987188 [0.999929,0.974848,0.997418,0.995275,0.960292,0.969438,0.999928,0.997533,0.99213,0.985866] + + area_under_roc_curve + 0.987588 [1,0.987189,0.989098,0.988067,0.96208,0.98696,1,0.996206,0.99213,0.974185] + + area_under_roc_curve + 0.993119 [1,0.996648,0.989977,0.998089,0.990401,0.998412,0.989801,0.998765,0.994636,0.974308] + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + build_cpu_time + 0.058 + + build_cpu_time + 0.058 + + build_cpu_time + 0.055 + + build_cpu_time + 0.054 + + build_cpu_time + 0.052 + + build_cpu_time + 0.052 + + build_cpu_time + 0.051 + + build_cpu_time + 0.054 + + build_cpu_time + 0.052 + + build_cpu_time + 0.052 + + f_measure + 0.922001 [1,0.894737,0.954128,0.910714,0.902655,0.953271,0.947368,0.932203,0.902655,0.824561] + + f_measure + 0.933066 [0.990826,0.93578,0.929825,0.929825,0.921739,0.914286,0.99115,0.954955,0.910714,0.852459] + + f_measure + 0.921167 [0.990826,0.900901,0.915888,0.912281,0.932203,0.93578,0.990991,0.933333,0.923077,0.777778] + + f_measure + 0.92075 [0.990826,0.854701,0.914286,0.944444,0.932203,0.924528,1,0.957265,0.87931,0.810345] + + f_measure + 0.928674 [0.981481,0.882883,0.928571,0.974359,0.949153,0.915888,0.963636,0.966102,0.915888,0.810345] + + f_measure + 0.913056 [0.981481,0.907563,0.934579,0.912281,0.928571,0.836735,0.973451,0.973913,0.839286,0.84127] + + f_measure + 0.931169 [0.972973,0.93913,0.971963,0.914286,0.923077,0.910714,0.972477,0.905983,0.927273,0.876033] + + f_measure + 0.909739 [0.99115,0.841121,0.936937,0.928571,0.899083,0.914286,0.982143,0.898305,0.892857,0.816] + + f_measure + 0.906202 [0.990991,0.898305,0.915888,0.9,0.915888,0.903846,1,0.883333,0.864865,0.789474] + + f_measure + 0.939667 [1,0.931034,0.947368,0.915888,0.972973,0.963636,0.982143,0.929825,0.894737,0.859649] + + kappa + 0.91301 + + kappa + 0.924872 + + kappa + 0.913007 + + kappa + 0.911031 + + kappa + 0.92091 + + kappa + 0.903112 + + kappa + 0.922892 + + kappa + 0.899172 + + kappa + 0.895209 + + kappa + 0.932781 + + kb_relative_information_score + 520.530219 + + kb_relative_information_score + 522.44632 + + kb_relative_information_score + 516.098237 + + kb_relative_information_score + 516.223961 + + kb_relative_information_score + 520.828521 + + kb_relative_information_score + 512.802292 + + kb_relative_information_score + 522.721047 + + kb_relative_information_score + 510.349442 + + kb_relative_information_score + 511.383119 + + kb_relative_information_score + 528.034273 + + mean_absolute_error + 0.01561 + + mean_absolute_error + 0.014946 + + mean_absolute_error + 0.017004 + + mean_absolute_error + 0.01725 + + mean_absolute_error + 0.014984 + + mean_absolute_error + 0.018277 + + mean_absolute_error + 0.014778 + + mean_absolute_error + 0.019306 + + mean_absolute_error + 0.018655 + + mean_absolute_error + 0.01293 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179998 + + mean_prior_absolute_error + 0.179998 + + mean_prior_absolute_error + 0.179998 + + mean_prior_absolute_error + 0.179999 + + number_of_instances + 562 [55,57,56,58,57,56,56,56,55,56] + + number_of_instances + 562 [55,57,56,58,57,56,56,56,55,56] + + number_of_instances + 562 [55,57,56,57,57,56,56,57,55,56] + + number_of_instances + 562 [55,57,56,57,57,56,56,57,55,56] + + number_of_instances + 562 [55,57,56,57,57,55,56,57,55,57] + + number_of_instances + 562 [55,57,56,57,57,55,56,57,55,57] + + number_of_instances + 562 [56,57,55,57,57,56,55,57,56,56] + + number_of_instances + 562 [56,57,55,57,57,56,55,57,56,56] + + number_of_instances + 562 [56,58,55,57,56,56,56,56,56,56] + + number_of_instances + 562 [56,57,56,57,56,56,56,56,56,56] + + precision + 0.923823 [1,0.894737,0.981132,0.944444,0.910714,1,0.931034,0.887097,0.87931,0.810345] + + precision + 0.936344 [1,0.980769,0.913793,0.946429,0.913793,0.979592,0.982456,0.963636,0.894737,0.787879] + + precision + 0.922887 [1,0.925926,0.960784,0.912281,0.901639,0.962264,1,0.888889,0.870968,0.807692] + + precision + 0.924699 [1,0.833333,0.979592,1,0.901639,0.98,1,0.933333,0.836066,0.783333] + + precision + 0.92969 [1,0.907407,0.928571,0.95,0.918033,0.942308,0.981481,0.934426,0.942308,0.79661] + + precision + 0.918297 [1,0.870968,0.980392,0.912281,0.945455,0.953488,0.964912,0.965517,0.824561,0.768116] + + precision + 0.934578 [0.981818,0.931034,1,1,0.9,0.910714,0.981481,0.883333,0.944444,0.815385] + + precision + 0.914296 [0.982456,0.9,0.928571,0.945455,0.942308,0.979592,0.964912,0.868852,0.892857,0.73913] + + precision + 0.909699 [1,0.883333,0.942308,0.857143,0.960784,0.979167,1,0.828125,0.872727,0.775862] + + precision + 0.94099 [1,0.915254,0.931034,0.98,0.981818,0.981481,0.982143,0.913793,0.87931,0.844828] + + predictive_accuracy + 0.921708 + + predictive_accuracy + 0.932384 + + predictive_accuracy + 0.921708 + + predictive_accuracy + 0.919929 + + predictive_accuracy + 0.928826 + + predictive_accuracy + 0.912811 + + predictive_accuracy + 0.930605 + + predictive_accuracy + 0.909253 + + predictive_accuracy + 0.905694 + + predictive_accuracy + 0.939502 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + recall + 0.921708 [1,0.894737,0.928571,0.87931,0.894737,0.910714,0.964286,0.982143,0.927273,0.839286] + + recall + 0.932384 [0.981818,0.894737,0.946429,0.913793,0.929825,0.857143,1,0.946429,0.927273,0.928571] + + recall + 0.921708 [0.981818,0.877193,0.875,0.912281,0.964912,0.910714,0.982143,0.982456,0.981818,0.75] + + recall + 0.919929 [0.981818,0.877193,0.857143,0.894737,0.964912,0.875,1,0.982456,0.927273,0.839286] + + recall + 0.928826 [0.963636,0.859649,0.928571,1,0.982456,0.890909,0.946429,1,0.890909,0.824561] + + recall + 0.912811 [0.963636,0.947368,0.892857,0.912281,0.912281,0.745455,0.982143,0.982456,0.854545,0.929825] + + recall + 0.930605 [0.964286,0.947368,0.945455,0.842105,0.947368,0.910714,0.963636,0.929825,0.910714,0.946429] + + recall + 0.909253 [1,0.789474,0.945455,0.912281,0.859649,0.857143,1,0.929825,0.892857,0.910714] + + recall + 0.905694 [0.982143,0.913793,0.890909,0.947368,0.875,0.839286,1,0.946429,0.857143,0.803571] + + recall + 0.939502 [1,0.947368,0.964286,0.859649,0.964286,0.946429,0.982143,0.946429,0.910714,0.875] + + relative_absolute_error + 0.086724 + + relative_absolute_error + 0.083036 + + relative_absolute_error + 0.094469 + + relative_absolute_error + 0.095836 + + relative_absolute_error + 0.083244 + + relative_absolute_error + 0.101539 + + relative_absolute_error + 0.082103 + + relative_absolute_error + 0.107256 + + relative_absolute_error + 0.103639 + + relative_absolute_error + 0.071835 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299998 + + root_mean_prior_squared_error + 0.299998 + + root_mean_prior_squared_error + 0.299999 + + root_mean_prior_squared_error + 0.3 + + root_mean_squared_error + 0.114072 + + root_mean_squared_error + 0.111221 + + root_mean_squared_error + 0.121483 + + root_mean_squared_error + 0.119208 + + root_mean_squared_error + 0.113177 + + root_mean_squared_error + 0.123617 + + root_mean_squared_error + 0.111738 + + root_mean_squared_error + 0.128468 + + root_mean_squared_error + 0.126509 + + root_mean_squared_error + 0.101792 + + root_relative_squared_error + 0.380244 + + root_relative_squared_error + 0.370741 + + root_relative_squared_error + 0.404947 + + root_relative_squared_error + 0.397364 + + root_relative_squared_error + 0.377262 + + root_relative_squared_error + 0.412061 + + root_relative_squared_error + 0.372463 + + root_relative_squared_error + 0.428228 + + root_relative_squared_error + 0.4217 + + root_relative_squared_error + 0.339306 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + diff --git a/tests/files/org/openml/test/setups/1/description.xml b/tests/files/org/openml/test/setups/1/description.xml new file mode 100644 index 000000000..ee234e4ff --- /dev/null +++ b/tests/files/org/openml/test/setups/1/description.xml @@ -0,0 +1,23 @@ + + 100 + 60 + + 3432 + 60 + weka.J48(1)_C + C + option + 0.25 + 0.9 + + + 3435 + 60 + weka.J48(1)_M + M + option + 2 + 2 + + + diff --git a/tests/files/tasks/1/datasplits.arff b/tests/files/org/openml/test/tasks/1/datasplits.arff similarity index 100% rename from tests/files/tasks/1/datasplits.arff rename to tests/files/org/openml/test/tasks/1/datasplits.arff diff --git a/tests/files/tasks/1/task.xml b/tests/files/org/openml/test/tasks/1/task.xml similarity index 100% rename from tests/files/tasks/1/task.xml rename to tests/files/org/openml/test/tasks/1/task.xml diff --git a/tests/files/tasks/1882/datasplits.arff b/tests/files/org/openml/test/tasks/1882/datasplits.arff similarity index 100% rename from tests/files/tasks/1882/datasplits.arff rename to tests/files/org/openml/test/tasks/1882/datasplits.arff diff --git a/tests/files/tasks/1882/task.xml b/tests/files/org/openml/test/tasks/1882/task.xml similarity index 100% rename from tests/files/tasks/1882/task.xml rename to tests/files/org/openml/test/tasks/1882/task.xml diff --git a/tests/files/tasks/3/datasplits.arff b/tests/files/org/openml/test/tasks/3/datasplits.arff similarity index 100% rename from tests/files/tasks/3/datasplits.arff rename to tests/files/org/openml/test/tasks/3/datasplits.arff diff --git a/tests/files/tasks/3/task.xml b/tests/files/org/openml/test/tasks/3/task.xml similarity index 100% rename from tests/files/tasks/3/task.xml rename to tests/files/org/openml/test/tasks/3/task.xml diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 0b11f3d73..5ec6c816b 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -1,6 +1,7 @@ import numpy as np from scipy import sparse import six +from time import time from openml.testing import TestBase import openml @@ -53,7 +54,9 @@ def test_get_data_with_target(self): self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (898, 38)) X, y, attribute_names = self.dataset.get_data( - target="class", return_attribute_names=True) + target="class", + return_attribute_names=True + ) self.assertEqual(len(attribute_names), 38) self.assertNotIn("class", attribute_names) self.assertEqual(y.shape, (898, )) @@ -61,13 +64,18 @@ def test_get_data_with_target(self): def test_get_data_rowid_and_ignore_and_target(self): self.dataset.ignore_attributes = ["condition"] self.dataset.row_id_attribute = ["hardness"] - X, y = self.dataset.get_data(target="class", include_row_id=False, - include_ignore_attributes=False) + X, y = self.dataset.get_data( + target="class", + include_row_id=False, + include_ignore_attributes=False + ) self.assertEqual(X.dtype, np.float32) self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (898, 36)) X, y, categorical = self.dataset.get_data( - target="class", return_categorical_indicator=True) + target="class", + return_categorical_indicator=True, + ) self.assertEqual(len(categorical), 36) self.assertListEqual(categorical, [True] * 3 + [False] + [True] * 2 + [ False] + [True] * 23 + [False] * 3 + [True] * 3) @@ -90,6 +98,25 @@ def test_get_data_with_ignore_attributes(self): # TODO test multiple ignore attributes! +class OpenMLDatasetTestOnTestServer(TestBase): + def setUp(self): + super(OpenMLDatasetTestOnTestServer, self).setUp() + # longley, really small dataset + self.dataset = openml.datasets.get_dataset(125) + + def test_tagging(self): + tag = "testing_tag_{}_{}".format(self.id(), time()) + ds_list = openml.datasets.list_datasets(tag=tag) + self.assertEqual(len(ds_list), 0) + self.dataset.push_tag(tag) + ds_list = openml.datasets.list_datasets(tag=tag) + self.assertEqual(len(ds_list), 1) + self.assertIn(125, ds_list) + self.dataset.remove_tag(tag) + ds_list = openml.datasets.list_datasets(tag=tag) + self.assertEqual(len(ds_list), 0) + + class OpenMLDatasetTestSparse(TestBase): _multiprocess_can_split_ = True @@ -107,7 +134,9 @@ def test_get_sparse_dataset_with_target(self): self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (600, 20000)) X, y, attribute_names = self.sparse_dataset.get_data( - target="class", return_attribute_names=True) + target="class", + return_attribute_names=True, + ) self.assertTrue(sparse.issparse(X)) self.assertEqual(len(attribute_names), 20000) self.assertNotIn("class", attribute_names) @@ -170,15 +199,34 @@ def test_get_sparse_dataset_rowid_and_ignore_and_target(self): self.sparse_dataset.ignore_attributes = ["V256"] self.sparse_dataset.row_id_attribute = ["V512"] X, y = self.sparse_dataset.get_data( - target="class", include_row_id=False, - include_ignore_attributes=False) + target="class", + include_row_id=False, + include_ignore_attributes=False, + ) self.assertTrue(sparse.issparse(X)) self.assertEqual(X.dtype, np.float32) self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (600, 19998)) X, y, categorical = self.sparse_dataset.get_data( - target="class", return_categorical_indicator=True) + target="class", + return_categorical_indicator=True, + ) self.assertTrue(sparse.issparse(X)) self.assertEqual(len(categorical), 19998) self.assertListEqual(categorical, [False] * 19998) self.assertEqual(y.shape, (600, )) + + +class OpenMLDatasetQualityTest(TestBase): + def test__check_qualities(self): + qualities = [{'oml:name': 'a', 'oml:value': '0.5'}] + qualities = openml.datasets.dataset._check_qualities(qualities) + self.assertEqual(qualities['a'], 0.5) + + qualities = [{'oml:name': 'a', 'oml:value': 'null'}] + qualities = openml.datasets.dataset._check_qualities(qualities) + self.assertNotEqual(qualities['a'], qualities['a']) + + qualities = [{'oml:name': 'a', 'oml:value': None}] + qualities = openml.datasets.dataset._check_qualities(qualities) + self.assertNotEqual(qualities['a'], qualities['a']) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 2a0d6be83..24c2bb77c 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -1,6 +1,5 @@ import unittest import os -import os import sys if sys.version_info[0] >= 3: @@ -8,13 +7,20 @@ else: import mock + +import random +import six + from oslo_concurrency import lockutils + import scipy.sparse import openml from openml import OpenMLDataset -from openml.exceptions import OpenMLCacheException, PyOpenMLError +from openml.exceptions import OpenMLCacheException, PyOpenMLError, \ + OpenMLHashException, PrivateDatasetError from openml.testing import TestBase +from openml.utils import _tag_entity, _create_cache_directory_for_id from openml.datasets.functions import (_get_cached_dataset, _get_cached_dataset_features, @@ -23,7 +29,8 @@ _get_dataset_description, _get_dataset_arff, _get_dataset_features, - _get_dataset_qualities) + _get_dataset_qualities, + DATASETS_CACHE_DIR_NAME) class TestOpenMLDataset(TestBase): @@ -51,7 +58,7 @@ def _remove_pickle_files(self): pass def test__list_cached_datasets(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir cached_datasets = openml.datasets.functions._list_cached_datasets() self.assertIsInstance(cached_datasets, list) self.assertEqual(len(cached_datasets), 2) @@ -59,7 +66,7 @@ def test__list_cached_datasets(self): @mock.patch('openml.datasets.functions._list_cached_datasets') def test__get_cached_datasets(self, _list_cached_datasets_mock): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir _list_cached_datasets_mock.return_value = [-1, 2] datasets = _get_cached_datasets() self.assertIsInstance(datasets, dict) @@ -67,63 +74,117 @@ def test__get_cached_datasets(self, _list_cached_datasets_mock): self.assertIsInstance(list(datasets.values())[0], OpenMLDataset) def test__get_cached_dataset(self, ): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir dataset = _get_cached_dataset(2) features = _get_cached_dataset_features(2) qualities = _get_cached_dataset_qualities(2) self.assertIsInstance(dataset, OpenMLDataset) self.assertTrue(len(dataset.features) > 0) self.assertTrue(len(dataset.features) == len(features['oml:feature'])) - self.assertTrue(len(dataset.qualities) == len(qualities['oml:quality'])) + self.assertTrue(len(dataset.qualities) == len(qualities)) def test_get_cached_dataset_description(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir description = openml.datasets.functions._get_cached_dataset_description(2) self.assertIsInstance(description, dict) def test_get_cached_dataset_description_not_cached(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir self.assertRaisesRegexp(OpenMLCacheException, "Dataset description for " "dataset id 3 not cached", openml.datasets.functions._get_cached_dataset_description, 3) def test_get_cached_dataset_arff(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir description = openml.datasets.functions._get_cached_dataset_arff( dataset_id=2) self.assertIsInstance(description, str) def test_get_cached_dataset_arff_not_cached(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir self.assertRaisesRegexp(OpenMLCacheException, "ARFF file for " "dataset id 3 not cached", openml.datasets.functions._get_cached_dataset_arff, 3) + def _check_dataset(self, dataset): + self.assertEqual(type(dataset), dict) + self.assertGreaterEqual(len(dataset), 2) + self.assertIn('did', dataset) + self.assertIsInstance(dataset['did'], int) + self.assertIn('status', dataset) + self.assertIsInstance(dataset['status'], six.string_types) + self.assertIn(dataset['status'], ['in_preparation', 'active', + 'deactivated']) + def _check_datasets(self, datasets): + for did in datasets: + self._check_dataset(datasets[did]) + + def test_tag_untag_dataset(self): + tag = 'test_tag_%d' %random.randint(1, 1000000) + all_tags = _tag_entity('data', 1, tag) + self.assertTrue(tag in all_tags) + all_tags = _tag_entity('data', 1, tag, untag=True) + self.assertTrue(tag not in all_tags) + def test_list_datasets(self): # We can only perform a smoke test here because we test on dynamic # data from the internet... datasets = openml.datasets.list_datasets() # 1087 as the number of datasets on openml.org self.assertGreaterEqual(len(datasets), 100) - for did in datasets: - self._check_dataset(datasets[did]) + self._check_datasets(datasets) def test_list_datasets_by_tag(self): datasets = openml.datasets.list_datasets(tag='study_14') self.assertGreaterEqual(len(datasets), 100) - for did in datasets: - self._check_dataset(datasets[did]) + self._check_datasets(datasets) + + def test_list_datasets_by_size(self): + datasets = openml.datasets.list_datasets(size=10050) + self.assertGreaterEqual(len(datasets), 120) + self._check_datasets(datasets) + + def test_list_datasets_by_number_instances(self): + datasets = openml.datasets.list_datasets(number_instances="5..100") + self.assertGreaterEqual(len(datasets), 4) + self._check_datasets(datasets) + + def test_list_datasets_by_number_features(self): + datasets = openml.datasets.list_datasets(number_features="50..100") + self.assertGreaterEqual(len(datasets), 8) + self._check_datasets(datasets) + + def test_list_datasets_by_number_classes(self): + datasets = openml.datasets.list_datasets(number_classes="5") + self.assertGreaterEqual(len(datasets), 3) + self._check_datasets(datasets) + + def test_list_datasets_by_number_missing_values(self): + datasets = openml.datasets.list_datasets(number_missing_values="5..100") + self.assertGreaterEqual(len(datasets), 5) + self._check_datasets(datasets) + + def test_list_datasets_combined_filters(self): + datasets = openml.datasets.list_datasets(tag='study_14', number_instances="100..1000", number_missing_values="800..1000") + self.assertGreaterEqual(len(datasets), 1) + self._check_datasets(datasets) def test_list_datasets_paginate(self): size = 10 max = 100 for i in range(0, max, size): datasets = openml.datasets.list_datasets(offset=i, size=size) - self.assertGreaterEqual(size, len(datasets)) - for did in datasets: - self._check_dataset(datasets[did]) + self.assertEqual(size, len(datasets)) + self._check_datasets(datasets) + + def test_list_datasets_empty(self): + datasets = openml.datasets.list_datasets(tag='NoOneWouldUseThisTagAnyway') + if len(datasets) > 0: + raise ValueError('UnitTest Outdated, tag was already used (please remove)') + + self.assertIsInstance(datasets, dict) @unittest.skip('See https://github.com/openml/openml-python/issues/149') def test_check_datasets_active(self): @@ -171,6 +232,11 @@ def test_get_dataset(self): self.assertGreater(len(dataset.features), 1) self.assertGreater(len(dataset.qualities), 4) + # Issue324 Properly handle private datasets when trying to access them + openml.config.server = self.production_server + self.assertRaises(PrivateDatasetError, openml.datasets.get_dataset, 45) + + def test_get_dataset_with_string(self): dataset = openml.datasets.get_dataset(101) self.assertRaises(PyOpenMLError, dataset._get_arff, 'arff') @@ -195,12 +261,26 @@ def test__get_dataset_description(self): self.assertTrue(os.path.exists(description_xml_path)) def test__getarff_path_dataset_arff(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir description = openml.datasets.functions._get_cached_dataset_description(2) arff_path = _get_dataset_arff(self.workdir, description) self.assertIsInstance(arff_path, str) self.assertTrue(os.path.exists(arff_path)) + def test__getarff_md5_issue(self): + description = { + 'oml:id': 5, + 'oml:md5_checksum': 'abc', + 'oml:url': 'https://www.openml.org/data/download/61', + } + self.assertRaisesRegexp( + OpenMLHashException, + 'Checksum ad484452702105cbf3d30f8deaba39a9 of downloaded dataset 5 ' + 'is unequal to the checksum abc sent by the server.', + _get_dataset_arff, + self.workdir, description, + ) + def test__get_dataset_features(self): features = _get_dataset_features(self.workdir, 2) self.assertIsInstance(features, dict) @@ -210,14 +290,17 @@ def test__get_dataset_features(self): def test__get_dataset_qualities(self): # Only a smoke check qualities = _get_dataset_qualities(self.workdir, 2) - self.assertIsInstance(qualities, dict) + self.assertIsInstance(qualities, list) def test_deletion_of_cache_dir(self): # Simple removal - did_cache_dir = openml.datasets.functions.\ - _create_dataset_cache_directory(1) + did_cache_dir = openml.utils._create_cache_directory_for_id( + DATASETS_CACHE_DIR_NAME, 1, + ) self.assertTrue(os.path.exists(did_cache_dir)) - openml.datasets.functions._remove_dataset_cache_dir(did_cache_dir) + openml.utils._remove_cache_dir_for_id( + DATASETS_CACHE_DIR_NAME, did_cache_dir, + ) self.assertFalse(os.path.exists(did_cache_dir)) # Use _get_dataset_arff to load the description, trigger an exception in the @@ -227,7 +310,9 @@ def test_deletion_of_cache_dir_faulty_download(self, patch): patch.side_effect = Exception('Boom!') self.assertRaisesRegexp(Exception, 'Boom!', openml.datasets.get_dataset, 1) - datasets_cache_dir = os.path.join(self.workdir, 'datasets') + datasets_cache_dir = os.path.join( + self.workdir, 'org', 'openml', 'test', 'datasets' + ) self.assertEqual(len(os.listdir(datasets_cache_dir)), 0) def test_publish_dataset(self): @@ -241,7 +326,7 @@ def test_publish_dataset(self): self.assertIsInstance(dataset.dataset_id, int) def test__retrieve_class_labels(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir labels = openml.datasets.get_dataset(2).retrieve_class_labels() self.assertEqual(labels, ['1', '2', '3', '4', '5', 'U']) labels = openml.datasets.get_dataset(2).retrieve_class_labels( diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 47e6d72e4..be55c2cd8 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -63,3 +63,10 @@ def test_evaluation_list_limit(self): evaluations = openml.evaluations.list_evaluations("predictive_accuracy", size=100, offset=100) self.assertEquals(len(evaluations), 100) + + def test_list_evaluations_empty(self): + evaluations = openml.evaluations.list_evaluations('unexisting_measure') + if len(evaluations) > 0: + raise ValueError('UnitTest Outdated, got somehow results') + + self.assertIsInstance(evaluations, dict) diff --git a/tests/test_examples/test_OpenMLDemo.py b/tests/test_examples/test_OpenMLDemo.py index 168978945..bdadcdbb2 100644 --- a/tests/test_examples/test_OpenMLDemo.py +++ b/tests/test_examples/test_OpenMLDemo.py @@ -2,12 +2,24 @@ import shutil import sys +import matplotlib +matplotlib.use('AGG') import nbformat -from nbconvert.preprocessors import ExecutePreprocessor -from nbconvert.preprocessors.execute import CellExecutionError +from nbconvert.exporters import export +from nbconvert.exporters.python import PythonExporter +import six +if six.PY2: + import mock +else: + import unittest.mock as mock + +import openml._api_calls +import openml.config from openml.testing import TestBase +_perform_api_call = openml._api_calls._perform_api_call + class OpenMLDemoTest(TestBase): def setUp(self): @@ -29,29 +41,39 @@ def setUp(self): except: pass - def _test_notebook(self, notebook_name): + def _tst_notebook(self, notebook_name): notebook_filename = os.path.abspath(os.path.join( self.this_file_directory, '..', '..', 'examples', notebook_name)) - notebook_filename_out = os.path.join( - self.notebook_output_directory, notebook_name) with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) - nb.metadata.get('kernelspec', {})['name'] = self.kernel_name - ep = ExecutePreprocessor(kernel_name=self.kernel_name) - ep.timeout = 60 - - try: - ep.preprocess(nb, {'metadata': {'path': self.this_file_directory}}) - except CellExecutionError as e: - msg = 'Error executing the notebook "%s". ' % notebook_filename - msg += 'See notebook "%s" for the traceback.\n\n' % notebook_filename_out - msg += e.traceback - self.fail(msg) - finally: - with open(notebook_filename_out, mode='wt') as f: - nbformat.write(nb, f) - - def test_tutorial(self): - self._test_notebook('OpenML_Tutorial.ipynb') + + python_nb, metadata = export(PythonExporter, nb) + + # Remove magic lines manually + python_nb = '\n'.join([ + line for line in python_nb.split('\n') + if 'get_ipython().run_line_magic(' not in line + ]) + + exec(python_nb) + + @mock.patch('openml._api_calls._perform_api_call') + def test_tutorial(self, patch): + def side_effect(*args, **kwargs): + if ( + args[0].endswith('/run/') + and kwargs['file_elements'] is not None + ): + return """ + 1 + + """ + else: + return _perform_api_call(*args, **kwargs) + patch.side_effect = side_effect + + openml.config.server = self.production_server + self._tst_notebook('OpenML_Tutorial.ipynb') + self.assertGreater(patch.call_count, 100) diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 303d890c6..54e3f28b1 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -36,8 +36,8 @@ class TestFlow(TestBase): _multiprocess_can_split_ = True def test_get_flow(self): - # We need to use the production server here because 4024 is not the test - # server + # We need to use the production server here because 4024 is not the + # test server openml.config.server = self.production_server flow = openml.flows.get_flow(4024) @@ -67,6 +67,21 @@ def test_get_flow(self): self.assertEqual(subflow_3.parameters['L'], '-1') self.assertEqual(len(subflow_3.components), 0) + def test_tagging(self): + flow_list = openml.flows.list_flows(size=1) + flow_id = list(flow_list.keys())[0] + flow = openml.flows.get_flow(flow_id) + tag = "testing_tag_{}_{}".format(self.id(), time.time()) + flow_list = openml.flows.list_flows(tag=tag) + self.assertEqual(len(flow_list), 0) + flow.push_tag(tag) + flow_list = openml.flows.list_flows(tag=tag) + self.assertEqual(len(flow_list), 1) + self.assertIn(flow_id, flow_list) + flow.remove_tag(tag) + flow_list = openml.flows.list_flows(tag=tag) + self.assertEqual(len(flow_list), 0) + def test_from_xml_to_xml(self): # Get the raw xml thing # TODO maybe get this via get_flow(), which would have to be refactored to allow getting only the xml dictionary @@ -182,7 +197,7 @@ def test_semi_legal_flow(self): flow.publish() @mock.patch('openml.flows.functions.get_flow') - @mock.patch('openml.flows.flow._perform_api_call') + @mock.patch('openml._api_calls._perform_api_call') def test_publish_error(self, api_call_mock, get_flow_mock): model = sklearn.ensemble.RandomForestClassifier() flow = openml.flows.sklearn_to_flow(model) diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 47e04581b..419b86f13 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -31,6 +31,13 @@ def test_list_flows(self): for fid in flows: self._check_flow(flows[fid]) + def test_list_flows_empty(self): + flows = openml.flows.list_flows(tag='NoOneEverUsesThisTag123') + if len(flows) > 0: + raise ValueError('UnitTest Outdated, got somehow results (please adapt)') + + self.assertIsInstance(flows, dict) + def test_list_flows_by_tag(self): flows = openml.flows.list_flows(tag='weka') self.assertGreaterEqual(len(flows), 5) diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index a97f49913..8be8a2bed 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -24,6 +24,7 @@ import sklearn.pipeline import sklearn.preprocessing import sklearn.tree +import sklearn.cluster import openml from openml.flows import OpenMLFlow, sklearn_to_flow, flow_to_sklearn @@ -100,6 +101,47 @@ def test_serialize_model(self, check_dependencies_mock): self.assertEqual(check_dependencies_mock.call_count, 1) + + @mock.patch('openml.flows.sklearn_converter._check_dependencies') + def test_serialize_model_clustering(self, check_dependencies_mock): + model = sklearn.cluster.KMeans() + + fixture_name = 'sklearn.cluster.k_means_.KMeans' + fixture_description = 'Automatically created scikit-learn flow.' + version_fixture = 'sklearn==%s\nnumpy>=1.6.1\nscipy>=0.9' \ + % sklearn.__version__ + fixture_parameters = \ + OrderedDict((('algorithm', '"auto"'), + ('copy_x', 'true'), + ('init', '"k-means++"'), + ('max_iter', '300'), + ('n_clusters', '8'), + ('n_init', '10'), + ('n_jobs', '1'), + ('precompute_distances', '"auto"'), + ('random_state', 'null'), + ('tol', '0.0001'), + ('verbose', '0'))) + + serialization = sklearn_to_flow(model) + + self.assertEqual(serialization.name, fixture_name) + self.assertEqual(serialization.class_name, fixture_name) + self.assertEqual(serialization.description, fixture_description) + self.assertEqual(serialization.parameters, fixture_parameters) + self.assertEqual(serialization.dependencies, version_fixture) + + new_model = flow_to_sklearn(serialization) + + self.assertEqual(type(new_model), type(model)) + self.assertIsNot(new_model, model) + + self.assertEqual(new_model.get_params(), model.get_params()) + new_model.fit(self.X) + + self.assertEqual(check_dependencies_mock.call_count, 1) + + def test_serialize_model_with_subcomponent(self): model = sklearn.ensemble.AdaBoostClassifier( n_estimators=100, base_estimator=sklearn.tree.DecisionTreeClassifier()) @@ -202,6 +244,64 @@ def test_serialize_pipeline(self): self.assertEqual(new_model_params, fu_params) new_model.fit(self.X, self.y) + def test_serialize_pipeline_clustering(self): + scaler = sklearn.preprocessing.StandardScaler(with_mean=False) + km = sklearn.cluster.KMeans() + model = sklearn.pipeline.Pipeline(steps=( + ('scaler', scaler), ('clusterer', km))) + + fixture_name = 'sklearn.pipeline.Pipeline(' \ + 'scaler=sklearn.preprocessing.data.StandardScaler,' \ + 'clusterer=sklearn.cluster.k_means_.KMeans)' + fixture_description = 'Automatically created scikit-learn flow.' + + serialization = sklearn_to_flow(model) + + self.assertEqual(serialization.name, fixture_name) + self.assertEqual(serialization.description, fixture_description) + + # Comparing the pipeline + # The parameters only have the name of base objects(not the whole flow) + # as value + self.assertEqual(len(serialization.parameters), 1) + # Hard to compare two representations of a dict due to possibly + # different sorting. Making a json makes it easier + self.assertEqual(json.loads(serialization.parameters['steps']), + [{'oml-python:serialized_object': + 'component_reference', 'value': {'key': 'scaler', 'step_name': 'scaler'}}, + {'oml-python:serialized_object': + 'component_reference', 'value': {'key': 'clusterer', 'step_name': 'clusterer'}}]) + + # Checking the sub-component + self.assertEqual(len(serialization.components), 2) + self.assertIsInstance(serialization.components['scaler'], + OpenMLFlow) + self.assertIsInstance(serialization.components['clusterer'], + OpenMLFlow) + + # del serialization.model + new_model = flow_to_sklearn(serialization) + + self.assertEqual(type(new_model), type(model)) + self.assertIsNot(new_model, model) + + self.assertEqual([step[0] for step in new_model.steps], + [step[0] for step in model.steps]) + self.assertIsNot(new_model.steps[0][1], model.steps[0][1]) + self.assertIsNot(new_model.steps[1][1], model.steps[1][1]) + + new_model_params = new_model.get_params() + del new_model_params['scaler'] + del new_model_params['clusterer'] + del new_model_params['steps'] + fu_params = model.get_params() + del fu_params['scaler'] + del fu_params['clusterer'] + del fu_params['steps'] + + self.assertEqual(new_model_params, fu_params) + new_model.fit(self.X, self.y) + def test_serialize_feature_union(self): ohe = sklearn.preprocessing.OneHotEncoder(sparse=False) scaler = sklearn.preprocessing.StandardScaler() @@ -597,4 +697,4 @@ def test_paralizable_check(self): self.assertTrue(_check_n_jobs(legal_models[i]) == answers[i]) for i in range(len(illegal_models)): - self.assertRaises(PyOpenMLError, _check_n_jobs, illegal_models[i]) \ No newline at end of file + self.assertRaises(PyOpenMLError, _check_n_jobs, illegal_models[i]) diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 2013f000e..deafbcacc 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -1,3 +1,5 @@ +from time import time + from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier from sklearn.linear_model import LogisticRegression from sklearn.model_selection import RandomizedSearchCV, StratifiedKFold @@ -5,39 +7,38 @@ from openml.testing import TestBase from openml.flows.sklearn_converter import sklearn_to_flow from openml import OpenMLRun +import openml class TestRun(TestBase): - # Splitting not helpful, these test's don't rely on the server and take less - # than 1 seconds + # Splitting not helpful, these test's don't rely on the server and take + # less than 1 seconds def test_parse_parameters_flow_not_on_server(self): model = LogisticRegression() flow = sklearn_to_flow(model) - self.assertRaisesRegexp(ValueError, - 'Flow sklearn.linear_model.logistic.LogisticRegression ' - 'has no flow_id!', - OpenMLRun._parse_parameters, flow) + self.assertRaisesRegexp( + ValueError, 'Flow sklearn.linear_model.logistic.LogisticRegression' + ' has no flow_id!', OpenMLRun._parse_parameters, flow) model = AdaBoostClassifier(base_estimator=LogisticRegression()) flow = sklearn_to_flow(model) flow.flow_id = 1 - self.assertRaisesRegexp(ValueError, - 'Flow sklearn.linear_model.logistic.LogisticRegression ' - 'has no flow_id!', - OpenMLRun._parse_parameters, flow) + self.assertRaisesRegexp( + ValueError, 'Flow sklearn.linear_model.logistic.LogisticRegression' + ' has no flow_id!', OpenMLRun._parse_parameters, flow) def test_parse_parameters(self): model = RandomizedSearchCV( estimator=RandomForestClassifier(n_estimators=5), - param_distributions={"max_depth": [3, None], - "max_features": [1, 2, 3, 4], - "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10], - "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - "bootstrap": [True, False], - "criterion": ["gini", "entropy"]}, + param_distributions={ + "max_depth": [3, None], + "max_features": [1, 2, 3, 4], + "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10], + "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "bootstrap": [True, False], "criterion": ["gini", "entropy"]}, cv=StratifiedKFold(n_splits=2, random_state=1), n_iter=5) flow = sklearn_to_flow(model) @@ -49,3 +50,19 @@ def test_parse_parameters(self): if parameter['oml:name'] == 'n_estimators': self.assertEqual(parameter['oml:value'], '5') self.assertEqual(parameter['oml:component'], 2) + + def test_tagging(self): + + runs = openml.runs.list_runs(size=1) + run_id = list(runs.keys())[0] + run = openml.runs.get_run(run_id) + tag = "testing_tag_{}_{}".format(self.id(), time()) + run_list = openml.runs.list_runs(tag=tag) + self.assertEqual(len(run_list), 0) + run.push_tag(tag) + run_list = openml.runs.list_runs(tag=tag) + self.assertEqual(len(run_list), 1) + self.assertIn(run_id, run_list) + run.remove_tag(tag) + run_list = openml.runs.list_runs(tag=tag) + self.assertEqual(len(run_list), 0) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 7606d3ac6..341900190 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -1,4 +1,5 @@ import arff +import collections import json import random import time @@ -26,6 +27,7 @@ from sklearn.feature_selection import VarianceThreshold from sklearn.linear_model import LogisticRegression, SGDClassifier, \ LinearRegression +from sklearn.neural_network import MLPClassifier from sklearn.ensemble import RandomForestClassifier, BaggingClassifier from sklearn.svm import SVC, LinearSVC from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, \ @@ -517,15 +519,21 @@ def test_get_run_trace(self): run = run.publish() self._wait_for_processed_run(run.run_id, 200) run_id = run.run_id - except openml.exceptions.PyOpenMLError: + except openml.exceptions.PyOpenMLError as e: + if 'Run already exists in server' not in e.message: + # in this case the error was not the one we expected + raise e # run was already flow = openml.flows.sklearn_to_flow(clf) flow_exists = openml.flows.flow_exists(flow.name, flow.external_version) self.assertIsInstance(flow_exists, int) + self.assertGreater(flow_exists, 0) downloaded_flow = openml.flows.get_flow(flow_exists) setup_exists = openml.setups.setup_exists(downloaded_flow) self.assertIsInstance(setup_exists, int) + self.assertGreater(setup_exists, 0) run_ids = _run_exists(task.task_id, setup_exists) + self.assertGreater(len(run_ids), 0) run_id = random.choice(list(run_ids)) # now the actual unit test ... @@ -613,18 +621,21 @@ def test__get_seeded_model_raises(self): self.assertRaises(ValueError, _get_seeded_model, model=clf, seed=42) def test__extract_arfftrace(self): - param_grid = {"max_depth": [3, None], - "max_features": [1, 2, 3, 4], - "bootstrap": [True, False], - "criterion": ["gini", "entropy"]} + param_grid = {"hidden_layer_sizes": [[5, 5], [10, 10], [20, 20]], + "activation" : ['identity', 'logistic', 'tanh', 'relu'], + "learning_rate_init": [0.1, 0.01, 0.001, 0.0001], + "max_iter": [10, 20, 40, 80]} num_iters = 10 task = openml.tasks.get_task(20) - clf = RandomizedSearchCV(RandomForestClassifier(), param_grid, num_iters) + clf = RandomizedSearchCV(MLPClassifier(), param_grid, num_iters) # just run the task train, _ = task.get_train_test_split_indices(0, 0) X, y = task.get_X_and_y() clf.fit(X[train], y[train]) + # check num layers of MLP + self.assertIn(clf.best_estimator_.hidden_layer_sizes, param_grid['hidden_layer_sizes']) + trace_attribute_list = _extract_arfftrace_attributes(clf) trace_list = _extract_arfftrace(clf, 0, 0) self.assertIsInstance(trace_attribute_list, list) @@ -658,7 +669,6 @@ def test__extract_arfftrace(self): else: # att_type = real self.assertIsInstance(trace_list[line_idx][att_idx], float) - self.assertEqual(set(param_grid.keys()), optimized_params) def test__prediction_to_row(self): @@ -714,13 +724,12 @@ def test_run_with_classifiers_in_param_grid(self): def test__run_task_get_arffcontent(self): task = openml.tasks.get_task(7) - class_labels = task.class_labels num_instances = 3196 num_folds = 10 num_repeats = 1 clf = SGDClassifier(loss='log', random_state=1) - res = openml.runs.functions._run_task_get_arffcontent(clf, task, class_labels) + res = openml.runs.functions._run_task_get_arffcontent(clf, task) arff_datacontent, arff_tracecontent, _, fold_evaluations, sample_evaluations = res # predictions self.assertIsInstance(arff_datacontent, list) @@ -748,6 +757,48 @@ def test__run_task_get_arffcontent(self): self.assertIn(arff_line[6], ['won', 'nowin']) self.assertIn(arff_line[7], ['won', 'nowin']) + def test__run_model_on_fold(self): + task = openml.tasks.get_task(7) + num_instances = 320 + num_folds = 1 + num_repeats = 1 + + clf = SGDClassifier(loss='log', random_state=1) + can_measure_runtime = sys.version_info[:2] >= (3, 3) + res = openml.runs.functions._run_model_on_fold(clf, task, 0, 0, 0, can_measure_runtime) + + arff_datacontent, arff_tracecontent, user_defined_measures, model = res + # predictions + self.assertIsInstance(arff_datacontent, list) + # trace. SGD does not produce any + self.assertIsInstance(arff_tracecontent, list) + self.assertEquals(len(arff_tracecontent), 0) + + fold_evaluations = collections.defaultdict(lambda: collections.defaultdict(dict)) + for measure in user_defined_measures: + fold_evaluations[measure][0][0] = user_defined_measures[measure] + + self._check_fold_evaluations(fold_evaluations, num_repeats, num_folds) + + # 10 times 10 fold CV of 150 samples + self.assertEqual(len(arff_datacontent), num_instances * num_repeats) + for arff_line in arff_datacontent: + # check number columns + self.assertEqual(len(arff_line), 8) + # check repeat + self.assertGreaterEqual(arff_line[0], 0) + self.assertLessEqual(arff_line[0], num_repeats - 1) + # check fold + self.assertGreaterEqual(arff_line[1], 0) + self.assertLessEqual(arff_line[1], num_folds - 1) + # check row id + self.assertGreaterEqual(arff_line[2], 0) + self.assertLessEqual(arff_line[2], num_instances - 1) + # check confidences + self.assertAlmostEqual(sum(arff_line[4:6]), 1.0) + self.assertIn(arff_line[6], ['won', 'nowin']) + self.assertIn(arff_line[7], ['won', 'nowin']) + def test__create_trace_from_arff(self): with open(self.static_cache_dir + '/misc/trace.arff', 'r') as arff_file: trace_arff = arff.load(arff_file) @@ -785,6 +836,13 @@ def test_get_runs_list(self): for rid in runs: self._check_run(runs[rid]) + def test_list_runs_empty(self): + runs = openml.runs.list_runs(task=[-1]) + if len(runs) > 0: + raise ValueError('UnitTest Outdated, got somehow results') + + self.assertIsInstance(runs, dict) + def test_get_runs_list_by_task(self): # TODO: comes from live, no such lists on test openml.config.server = self.production_server @@ -863,7 +921,11 @@ def test_get_runs_list_by_filters(self): uploaders_2 = [29, 274] flows = [74, 1718] - self.assertRaises(openml.exceptions.OpenMLServerError, openml.runs.list_runs) + ''' + Since the results are taken by batch size, the function does not throw an OpenMLServerError anymore. + Instead it throws a TimeOutException. For the moment commented out. + ''' + #self.assertRaises(openml.exceptions.OpenMLServerError, openml.runs.list_runs) runs = openml.runs.list_runs(id=ids) self.assertEqual(len(runs), 2) @@ -896,7 +958,7 @@ def test_run_on_dataset_with_missing_labels(self): model = Pipeline(steps=[('Imputer', Imputer(strategy='median')), ('Estimator', DecisionTreeClassifier())]) - data_content, _, _, _, _ = _run_task_get_arffcontent(model, task, class_labels) + data_content, _, _, _, _ = _run_task_get_arffcontent(model, task) # 2 folds, 5 repeats; keep in mind that this task comes from the test # server, the task on the live server is different self.assertEqual(len(data_content), 4490) @@ -917,8 +979,8 @@ def test_predict_proba_hardclassifier(self): ('imputer', sklearn.preprocessing.Imputer()), ('estimator', HardNaiveBayes()) ]) - arff_content1, arff_header1, _, _, _ = _run_task_get_arffcontent(clf1, task, task.class_labels) - arff_content2, arff_header2, _, _, _ = _run_task_get_arffcontent(clf2, task, task.class_labels) + arff_content1, arff_header1, _, _, _ = _run_task_get_arffcontent(clf1, task) + arff_content2, arff_header2, _, _, _ = _run_task_get_arffcontent(clf2, task) # verifies last two arff indices (predict and correct) # TODO: programmatically check wether these are indeed features (predict, correct) @@ -926,3 +988,12 @@ def test_predict_proba_hardclassifier(self): predictionsB = np.array(arff_content2)[:, -2:] np.testing.assert_array_equal(predictionsA, predictionsB) + + def test_get_cached_run(self): + openml.config.cache_directory = self.static_cache_dir + openml.runs.functions._get_cached_run(1) + + def test_get_uncached_run(self): + openml.config.cache_directory = self.static_cache_dir + with self.assertRaises(openml.exceptions.OpenMLCacheException): + openml.runs.functions._get_cached_run(10) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 5e77649b4..928874837 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -137,6 +137,13 @@ def test_setup_list_filter_flow(self): for setup_id in setups.keys(): self.assertEquals(setups[setup_id].flow_id, flow_id) + def test_list_setups_empty(self): + setups = openml.setups.list_setups(setup=[0]) + if len(setups) > 0: + raise ValueError('UnitTest Outdated, got somehow results') + + self.assertIsInstance(setups, dict) + def test_setuplist_offset(self): # TODO: remove after pull on live for better testing # openml.config.server = self.production_server @@ -149,4 +156,14 @@ def test_setuplist_offset(self): all = set(setups.keys()).union(setups2.keys()) - self.assertEqual(len(all), size * 2) \ No newline at end of file + self.assertEqual(len(all), size * 2) + + def test_get_cached_setup(self): + openml.config.cache_directory = self.static_cache_dir + openml.setups.functions._get_cached_setup(1) + + + def test_get_uncached_setup(self): + openml.config.cache_directory = self.static_cache_dir + with self.assertRaises(openml.exceptions.OpenMLCacheException): + openml.setups.functions._get_cached_setup(10) diff --git a/tests/test_study/test_study_examples.py b/tests/test_study/test_study_examples.py new file mode 100644 index 000000000..1dea4085c --- /dev/null +++ b/tests/test_study/test_study_examples.py @@ -0,0 +1,53 @@ +from openml.testing import TestBase + + +class TestStudyFunctions(TestBase): + _multiprocess_can_split_ = True + """Test the example code of Bischl et al. (2018)""" + + def test_Figure1a(self): + """Test listing in Figure 1a on a single task and the old OpenML100 study. + + The original listing is pasted into the comment below because it the + actual unit test differs a bit, as for example it does not run for all tasks, + but only a single one. + + import openml + import sklearn.tree, sklearn.preprocessing + benchmark_suite = openml.study.get_study('OpenML-CC18','tasks') # obtain the benchmark suite + clf = sklearn.pipeline.Pipeline(steps=[('imputer',sklearn.preprocessing.Imputer()), ('estimator',sklearn.tree.DecisionTreeClassifier())]) # build a sklearn classifier + for task_id in benchmark_suite.tasks: # iterate over all tasks + task = openml.tasks.get_task(task_id) # download the OpenML task + X, y = task.get_X_and_y() # get the data (not used in this example) + openml.config.apikey = 'FILL_IN_OPENML_API_KEY' # set the OpenML Api Key + run = openml.runs.run_model_on_task(task,clf) # run classifier on splits (requires API key) + score = run.get_metric_fn(sklearn.metrics.accuracy_score) # print accuracy score + print('Data set: %s; Accuracy: %0.2f' % (task.get_dataset().name,score.mean())) + run.publish() # publish the experiment on OpenML (optional) + print('URL for run: %s/run/%d' %(openml.config.server,run.run_id)) + """ + import openml + import sklearn.tree, sklearn.preprocessing + benchmark_suite = openml.study.get_study( + 'OpenML100', 'tasks' + ) # obtain the benchmark suite + clf = sklearn.pipeline.Pipeline( + steps=[ + ('imputer', sklearn.preprocessing.Imputer()), + ('estimator', sklearn.tree.DecisionTreeClassifier()) + ] + ) # build a sklearn classifier + for task_id in benchmark_suite.tasks[:1]: # iterate over all tasks + task = openml.tasks.get_task(task_id) # download the OpenML task + X, y = task.get_X_and_y() # get the data (not used in this example) + openml.config.apikey = openml.config.apikey # set the OpenML Api Key + run = openml.runs.run_model_on_task( + task, clf, + ) # run classifier on splits (requires API key) + score = run.get_metric_fn( + sklearn.metrics.accuracy_score + ) # print accuracy score + print('Data set: %s; Accuracy: %0.2f' % ( + task.get_dataset().name, score.mean())) + run.publish() # publish the experiment on OpenML (optional) + print('URL for run: %s/run/%d' % (openml.config.server, run.run_id)) diff --git a/tests/test_study/test_study_functions.py b/tests/test_study/test_study_functions.py index 0bf0496da..c2d0b7258 100644 --- a/tests/test_study/test_study_functions.py +++ b/tests/test_study/test_study_functions.py @@ -23,4 +23,4 @@ def test_get_tasks(self): self.assertEquals(study.data, None) self.assertGreater(len(study.tasks), 0) self.assertEquals(study.flows, None) - self.assertEquals(study.setups, None) \ No newline at end of file + self.assertEquals(study.setups, None) diff --git a/tests/test_tasks/test_split.py b/tests/test_tasks/test_split.py index e58e2dc2d..6fd2926e5 100644 --- a/tests/test_tasks/test_split.py +++ b/tests/test_tasks/test_split.py @@ -16,7 +16,9 @@ def setUp(self): self.directory = os.path.dirname(__file__) # This is for dataset self.arff_filename = os.path.join( - self.directory, "..", "files", "tasks", "1882", "datasplits.arff") + self.directory, "..", "files", "org", "openml", "test", + "tasks", "1882", "datasplits.arff" + ) self.pd_filename = self.arff_filename.replace(".arff", ".pkl") def tearDown(self): diff --git a/tests/test_tasks/test_task.py b/tests/test_tasks/test_task.py index 7b95d4cec..fdbfa06d1 100644 --- a/tests/test_tasks/test_task.py +++ b/tests/test_tasks/test_task.py @@ -1,11 +1,11 @@ import sys -import types if sys.version_info[0] >= 3: from unittest import mock else: import mock +from time import time import numpy as np import openml @@ -45,8 +45,21 @@ def test_get_X_and_Y(self): self.assertIsInstance(Y, np.ndarray) self.assertEqual(Y.dtype, float) + def test_tagging(self): + task = openml.tasks.get_task(1) + tag = "testing_tag_{}_{}".format(self.id(), time()) + task_list = openml.tasks.list_tasks(tag=tag) + self.assertEqual(len(task_list), 0) + task.push_tag(tag) + task_list = openml.tasks.list_tasks(tag=tag) + self.assertEqual(len(task_list), 1) + self.assertIn(1, task_list) + task.remove_tag(tag) + task_list = openml.tasks.list_tasks(tag=tag) + self.assertEqual(len(task_list), 0) + def test_get_train_and_test_split_indices(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir task = openml.tasks.get_task(1882) train_indices, test_indices = task.get_train_test_split_indices(0, 0) self.assertEqual(16, train_indices[0]) @@ -62,4 +75,3 @@ def test_get_train_and_test_split_indices(self): task.get_train_test_split_indices, 10, 0) self.assertRaisesRegexp(ValueError, "Repeat 10 not known", task.get_train_test_split_indices, 0, 10) - diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 0eba2b8a7..a711534c6 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -18,19 +18,19 @@ class TestTask(TestBase): _multiprocess_can_split_ = True def test__get_cached_tasks(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir tasks = openml.tasks.functions._get_cached_tasks() self.assertIsInstance(tasks, dict) self.assertEqual(len(tasks), 3) self.assertIsInstance(list(tasks.values())[0], OpenMLTask) def test__get_cached_task(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir task = openml.tasks.functions._get_cached_task(1) self.assertIsInstance(task, OpenMLTask) def test__get_cached_task_not_cached(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir self.assertRaisesRegexp(OpenMLCacheException, 'Task file for tid 2 not cached', openml.tasks.functions._get_cached_task, 2) @@ -42,6 +42,12 @@ def test__get_estimation_procedure_list(self): self.assertIsInstance(estimation_procedures[0], dict) self.assertEqual(estimation_procedures[0]['task_type_id'], 1) + def test_list_clustering_task(self): + # as shown by #383, clustering tasks can give list/dict casting problems + openml.config.server = self.production_server + openml.tasks.list_tasks(task_type_id=5, size=10) + # the expected outcome is that it doesn't crash. No assertions. + def _check_task(self, task): self.assertEqual(type(task), dict) self.assertGreaterEqual(len(task), 2) @@ -61,6 +67,13 @@ def test_list_tasks_by_type(self): self.assertEquals(ttid, tasks[tid]["ttid"]) self._check_task(tasks[tid]) + def test_list_tasks_empty(self): + tasks = openml.tasks.list_tasks(tag='NoOneWillEverUseThisTag') + if len(tasks) > 0: + raise ValueError('UnitTest Outdated, got somehow results (tag is used, please adapt)') + + self.assertIsInstance(tasks, dict) + def test_list_tasks_by_tag(self): num_basic_tasks = 100 # number is flexible, check server if fails tasks = openml.tasks.list_tasks(tag='study_14') @@ -96,18 +109,25 @@ def test_list_tasks_per_type_paginate(self): self._check_task(tasks[tid]) def test__get_task(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir task = openml.tasks.get_task(1882) + # Test the following task as it used to throw an Unicode Error. + # https://github.com/openml/openml-python/issues/378 + openml.config.server = self.production_server + production_task = openml.tasks.get_task(34536) def test_get_task(self): task = openml.tasks.get_task(1) self.assertIsInstance(task, OpenMLTask) - self.assertTrue(os.path.exists( - os.path.join(os.getcwd(), "tasks", "1", "task.xml"))) - self.assertTrue(os.path.exists( - os.path.join(os.getcwd(), "tasks", "1", "datasplits.arff"))) - self.assertTrue(os.path.exists( - os.path.join(os.getcwd(), "datasets", "1", "dataset.arff"))) + self.assertTrue(os.path.exists(os.path.join( + self.workdir, 'org', 'openml', 'test', "tasks", "1", "task.xml", + ))) + self.assertTrue(os.path.exists(os.path.join( + self.workdir, 'org', 'openml', 'test', "tasks", "1", "datasplits.arff" + ))) + self.assertTrue(os.path.exists(os.path.join( + self.workdir, 'org', 'openml', 'test', "datasets", "1", "dataset.arff" + ))) @mock.patch('openml.tasks.functions.get_dataset') def test_removal_upon_download_failure(self, get_dataset): @@ -127,9 +147,8 @@ def assert_and_raise(*args, **kwargs): os.path.join(os.getcwd(), "tasks", "1", "tasks.xml") )) - def test_get_task_with_cache(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir task = openml.tasks.get_task(1) self.assertIsInstance(task, OpenMLTask) @@ -137,13 +156,15 @@ def test_download_split(self): task = openml.tasks.get_task(1) split = task.download_split() self.assertEqual(type(split), OpenMLSplit) - self.assertTrue(os.path.exists( - os.path.join(os.getcwd(), "tasks", "1", "datasplits.arff"))) + self.assertTrue(os.path.exists(os.path.join( + self.workdir, 'org', 'openml', 'test', "tasks", "1", "datasplits.arff" + ))) def test_deletion_of_cache_dir(self): # Simple removal - tid_cache_dir = openml.tasks.functions.\ - _create_task_cache_directory(1) + tid_cache_dir = openml.utils._create_cache_directory_for_id( + 'tasks', 1, + ) self.assertTrue(os.path.exists(tid_cache_dir)) - openml.tasks.functions._remove_task_cache_dir(tid_cache_dir) + openml.utils._remove_cache_dir_for_id('tasks', tid_cache_dir) self.assertFalse(os.path.exists(tid_cache_dir)) diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py new file mode 100644 index 000000000..9c5274810 --- /dev/null +++ b/tests/test_utils/test_utils.py @@ -0,0 +1,18 @@ +from openml.testing import TestBase +import openml + + +class OpenMLTaskTest(TestBase): + _multiprocess_can_split_ = True + + def test_list_all(self): + list_datasets = openml.datasets.functions._list_datasets + datasets = openml.utils.list_all(list_datasets) + + self.assertGreaterEqual(len(datasets), 100) + for did in datasets: + self._check_dataset(datasets[did]) + + # TODO implement these tests + # datasets = openml.utils.list_all(list_datasets, limit=50) + # self.assertEqual(len(datasets), 50) \ No newline at end of file