From 8262c0e8ab1445ba001ad4ca6e29d8d0da8e130f Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sun, 10 Sep 2017 10:02:53 +0200 Subject: [PATCH 01/86] can read description files --- openml/runs/functions.py | 79 ++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 505cb9101..73d039464 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -588,20 +588,20 @@ def _create_run_from_xml(xml): New run object representing run_xml. """ run = xmltodict.parse(xml)["oml:run"] - run_id = int(run['oml:run_id']) - uploader = int(run['oml:uploader']) - uploader_name = run['oml:uploader_name'] + run_id = int(run['oml:run_id']) if 'oml:run_id' in run else None + uploader = int(run['oml:uploader']) if 'oml:uploader' in run else None + uploader_name = run['oml:uploader_name'] if 'oml:uploader_name' in run else None task_id = int(run['oml:task_id']) - task_type = run['oml:task_type'] + task_type = run['oml:task_type'] if 'oml:task_type' in run else None if 'oml:task_evaluation_measure' in run: task_evaluation_measure = run['oml:task_evaluation_measure'] else: task_evaluation_measure = None flow_id = int(run['oml:flow_id']) - flow_name = run['oml:flow_name'] - setup_id = int(run['oml:setup_id']) - setup_string = run['oml:setup_string'] + flow_name = run['oml:flow_name'] if 'oml:flow_name' in run else None + setup_id = int(run['oml:setup_id']) if 'oml:setup_id' in run else None + setup_string = run['oml:setup_string'] if 'oml:setup_string' in run else None parameters = dict() if 'oml:parameter_settings' in run: @@ -611,7 +611,7 @@ def _create_run_from_xml(xml): value = parameter_dict['oml:value'] parameters[key] = value - dataset_id = int(run['oml:input_data']['oml:dataset']['oml:did']) + dataset_id = int(run['oml:input_data']['oml:dataset']['oml:did']) if 'oml:input_data' in run else None files = dict() evaluations = dict() @@ -619,7 +619,8 @@ def _create_run_from_xml(xml): sample_evaluations = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) if 'oml:output_data' not in run: raise ValueError('Run does not contain output_data (OpenML server error?)') - else: + + if 'oml:file' in 'oml:output_data': if isinstance(run['oml:output_data']['oml:file'], dict): # only one result.. probably due to an upload error file_dict = run['oml:output_data']['oml:file'] @@ -631,40 +632,40 @@ def _create_run_from_xml(xml): else: raise TypeError(type(run['oml:output_data']['oml:file'])) - if 'oml:evaluation' in run['oml:output_data']: - # in normal cases there should be evaluations, but in case there - # was an error these could be absent - for evaluation_dict in run['oml:output_data']['oml:evaluation']: - key = evaluation_dict['oml:name'] - if 'oml:value' in evaluation_dict: - value = float(evaluation_dict['oml:value']) - elif 'oml:array_data' in evaluation_dict: - value = evaluation_dict['oml:array_data'] - else: - raise ValueError('Could not find keys "value" or "array_data" ' - 'in %s' % str(evaluation_dict.keys())) - if '@repeat' in evaluation_dict and '@fold' in evaluation_dict and '@sample' in evaluation_dict: - repeat = int(evaluation_dict['@repeat']) - fold = int(evaluation_dict['@fold']) - sample = int(evaluation_dict['@sample']) - repeat_dict = sample_evaluations[key] - fold_dict = repeat_dict[repeat] - sample_dict = fold_dict[fold] - sample_dict[sample] = value - elif '@repeat' in evaluation_dict and '@fold' in evaluation_dict: - repeat = int(evaluation_dict['@repeat']) - fold = int(evaluation_dict['@fold']) - repeat_dict = fold_evaluations[key] - fold_dict = repeat_dict[repeat] - fold_dict[fold] = value - else: - evaluations[key] = value + if 'oml:evaluation' in run['oml:output_data']: + # in normal cases there should be evaluations, but in case there + # was an error these could be absent + for evaluation_dict in run['oml:output_data']['oml:evaluation']: + key = evaluation_dict['oml:name'] + if 'oml:value' in evaluation_dict: + value = float(evaluation_dict['oml:value']) + elif 'oml:array_data' in evaluation_dict: + value = evaluation_dict['oml:array_data'] + else: + raise ValueError('Could not find keys "value" or "array_data" ' + 'in %s' % str(evaluation_dict.keys())) + if '@repeat' in evaluation_dict and '@fold' in evaluation_dict and '@sample' in evaluation_dict: + repeat = int(evaluation_dict['@repeat']) + fold = int(evaluation_dict['@fold']) + sample = int(evaluation_dict['@sample']) + repeat_dict = sample_evaluations[key] + fold_dict = repeat_dict[repeat] + sample_dict = fold_dict[fold] + sample_dict[sample] = value + elif '@repeat' in evaluation_dict and '@fold' in evaluation_dict: + repeat = int(evaluation_dict['@repeat']) + fold = int(evaluation_dict['@fold']) + repeat_dict = fold_evaluations[key] + fold_dict = repeat_dict[repeat] + fold_dict[fold] = value + else: + evaluations[key] = value - if 'description' not in files: + if 'description' not in files and run_id is not None: raise ValueError('No description file for run %d in run ' 'description XML' % run_id) - if 'predictions' not in files: + if 'predictions' not in files and run_id is not None: # JvR: actually, I am not sure whether this error should be raised. # a run can consist without predictions. But for now let's keep it raise ValueError('No prediction files for run %d in run ' From b284931aadefe6e3e3cc1c8477635b8514ff6066 Mon Sep 17 00:00:00 2001 From: toon Date: Tue, 10 Oct 2017 15:15:54 +0200 Subject: [PATCH 02/86] clustrering unit test --- tests/test_flows/test_sklearn.py | 44 +++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index a97f49913..5a1f688be 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -24,6 +24,7 @@ import sklearn.pipeline import sklearn.preprocessing import sklearn.tree +import sklearn.cluster import openml from openml.flows import OpenMLFlow, sklearn_to_flow, flow_to_sklearn @@ -100,6 +101,47 @@ def test_serialize_model(self, check_dependencies_mock): self.assertEqual(check_dependencies_mock.call_count, 1) + + @mock.patch('openml.flows.sklearn_converter._check_dependencies') + def test_serialize_model_clustering(self, check_dependencies_mock): + model = sklearn.cluster.KMeans() + + fixture_name = 'sklearn.cluster.k_means_.KMeans' + fixture_description = 'Automatically created scikit-learn flow.' + version_fixture = 'sklearn==%s\nnumpy>=1.6.1\nscipy>=0.9' \ + % sklearn.__version__ + fixture_parameters = \ + OrderedDict((('algorithm', '"auto"'), + ('copy_x', 'true'), + ('init', '"k-means++"'), + ('max_iter', '300'), + ('n_clusters', '8'), + ('n_init', '10'), + ('n_jobs', '1'), + ('precompute_distances', '"auto"'), + ('random_state', 'null'), + ('tol', '0.0001'), + ('verbose', '0'))) + + serialization = sklearn_to_flow(model) + + self.assertEqual(serialization.name, fixture_name) + self.assertEqual(serialization.class_name, fixture_name) + self.assertEqual(serialization.description, fixture_description) + self.assertEqual(serialization.parameters, fixture_parameters) + self.assertEqual(serialization.dependencies, version_fixture) + + new_model = flow_to_sklearn(serialization) + + self.assertEqual(type(new_model), type(model)) + self.assertIsNot(new_model, model) + + self.assertEqual(new_model.get_params(), model.get_params()) + new_model.fit(self.X) + + self.assertEqual(check_dependencies_mock.call_count, 1) + + def test_serialize_model_with_subcomponent(self): model = sklearn.ensemble.AdaBoostClassifier( n_estimators=100, base_estimator=sklearn.tree.DecisionTreeClassifier()) @@ -597,4 +639,4 @@ def test_paralizable_check(self): self.assertTrue(_check_n_jobs(legal_models[i]) == answers[i]) for i in range(len(illegal_models)): - self.assertRaises(PyOpenMLError, _check_n_jobs, illegal_models[i]) \ No newline at end of file + self.assertRaises(PyOpenMLError, _check_n_jobs, illegal_models[i]) From da5bb80b06355326ab8a2e8ae1124a90b0877f21 Mon Sep 17 00:00:00 2001 From: toon Date: Thu, 12 Oct 2017 10:16:00 +0200 Subject: [PATCH 03/86] added a unit test for a clustering pipeline --- tests/test_flows/test_sklearn.py | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index 5a1f688be..8be8a2bed 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -244,6 +244,64 @@ def test_serialize_pipeline(self): self.assertEqual(new_model_params, fu_params) new_model.fit(self.X, self.y) + def test_serialize_pipeline_clustering(self): + scaler = sklearn.preprocessing.StandardScaler(with_mean=False) + km = sklearn.cluster.KMeans() + model = sklearn.pipeline.Pipeline(steps=( + ('scaler', scaler), ('clusterer', km))) + + fixture_name = 'sklearn.pipeline.Pipeline(' \ + 'scaler=sklearn.preprocessing.data.StandardScaler,' \ + 'clusterer=sklearn.cluster.k_means_.KMeans)' + fixture_description = 'Automatically created scikit-learn flow.' + + serialization = sklearn_to_flow(model) + + self.assertEqual(serialization.name, fixture_name) + self.assertEqual(serialization.description, fixture_description) + + # Comparing the pipeline + # The parameters only have the name of base objects(not the whole flow) + # as value + self.assertEqual(len(serialization.parameters), 1) + # Hard to compare two representations of a dict due to possibly + # different sorting. Making a json makes it easier + self.assertEqual(json.loads(serialization.parameters['steps']), + [{'oml-python:serialized_object': + 'component_reference', 'value': {'key': 'scaler', 'step_name': 'scaler'}}, + {'oml-python:serialized_object': + 'component_reference', 'value': {'key': 'clusterer', 'step_name': 'clusterer'}}]) + + # Checking the sub-component + self.assertEqual(len(serialization.components), 2) + self.assertIsInstance(serialization.components['scaler'], + OpenMLFlow) + self.assertIsInstance(serialization.components['clusterer'], + OpenMLFlow) + + # del serialization.model + new_model = flow_to_sklearn(serialization) + + self.assertEqual(type(new_model), type(model)) + self.assertIsNot(new_model, model) + + self.assertEqual([step[0] for step in new_model.steps], + [step[0] for step in model.steps]) + self.assertIsNot(new_model.steps[0][1], model.steps[0][1]) + self.assertIsNot(new_model.steps[1][1], model.steps[1][1]) + + new_model_params = new_model.get_params() + del new_model_params['scaler'] + del new_model_params['clusterer'] + del new_model_params['steps'] + fu_params = model.get_params() + del fu_params['scaler'] + del fu_params['clusterer'] + del fu_params['steps'] + + self.assertEqual(new_model_params, fu_params) + new_model.fit(self.X, self.y) + def test_serialize_feature_union(self): ohe = sklearn.preprocessing.OneHotEncoder(sparse=False) scaler = sklearn.preprocessing.StandardScaler() From ca029b85fb507ce81ee283fb706f07d197b464d6 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Thu, 12 Oct 2017 10:51:54 +0200 Subject: [PATCH 04/86] add task ids to docs of list_tasks --- openml/tasks/functions.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 7245e9ddf..7400dc0c8 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -94,6 +94,16 @@ def list_tasks(task_type_id=None, offset=None, size=None, tag=None): task_type_id : int, optional ID of the task type as detailed `here `_. + + - Supervised classification: 1 + - Supervised regression: 2 + - Learning curve: 3 + - Supervised data stream classification: 4 + - Clustering: 5 + - Machine Learning Challenge: 6 + - Survival Analysis: 7 + - Subgroup Discovery: 8 + offset : int, optional the number of tasks to skip, starting from the first size : int, optional From 03c995521f7b6b01214d591782597b04c3cb6d1d Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Thu, 12 Oct 2017 11:15:50 +0200 Subject: [PATCH 05/86] fix docstrings in list_datasets, some pep8 # Conflicts: # openml/datasets/functions.py --- openml/datasets/functions.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 8e37d02ef..4a5c2ff7e 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -44,8 +44,8 @@ def _list_cached_datasets(): directory_name) dataset_directory_content = os.listdir(directory_name) - if "dataset.arff" in dataset_directory_content and \ - "description.xml" in dataset_directory_content: + if ("dataset.arff" in dataset_directory_content and + "description.xml" in dataset_directory_content): if dataset_id not in datasets: datasets.append(dataset_id) @@ -142,11 +142,11 @@ def list_datasets(offset=None, size=None, tag=None): Parameters ---------- offset : int, optional - the number of datasets to skip, starting from the first + The number of datasets to skip, starting from the first. size : int, optional - the maximum datasets of tasks to show + The maximum number of datasets to show. tag : str, optional - the tag to include + Only include datasets matching this tag. Returns ------- @@ -168,7 +168,7 @@ def list_datasets(offset=None, size=None, tag=None): api_call += "/offset/%d" % int(offset) if size is not None: - api_call += "/limit/%d" % int(size) + api_call += "/limit/%d" % int(size) if tag is not None: api_call += "/tag/%s" % tag @@ -185,7 +185,7 @@ def _list_datasets(api_call): assert type(datasets_dict['oml:data']['oml:dataset']) == list, \ type(datasets_dict['oml:data']) assert datasets_dict['oml:data']['@xmlns:oml'] == \ - 'http://openml.org/openml', datasets_dict['oml:data']['@xmlns:oml'] + 'http://openml.org/openml', datasets_dict['oml:data']['@xmlns:oml'] datasets = dict() for dataset_ in datasets_dict['oml:data']['oml:dataset']: @@ -289,7 +289,6 @@ def get_dataset(dataset_id): description = _get_dataset_description(did_cache_dir, dataset_id) arff_file = _get_dataset_arff(did_cache_dir, description) features = _get_dataset_features(did_cache_dir, dataset_id) - # TODO not used yet, figure out what to do with this... qualities = _get_dataset_qualities(did_cache_dir, dataset_id) except Exception as e: _remove_dataset_cache_dir(did_cache_dir) @@ -480,7 +479,8 @@ def _create_dataset_cache_directory(dataset_id): str Path of the created dataset cache directory. """ - dataset_cache_dir = os.path.join(config.get_cache_directory(), "datasets", str(dataset_id)) + dataset_cache_dir = os.path.join(config.get_cache_directory(), "datasets", + str(dataset_id)) try: os.makedirs(dataset_cache_dir) except (OSError, IOError): From 886a2175138fdaec0f9352aa22681cbafa0673ce Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 12 Oct 2017 11:45:10 +0200 Subject: [PATCH 06/86] FIX #197, do not automatically cast target attribute --- openml/datasets/dataset.py | 8 +++++- openml/tasks/task.py | 16 +++++++----- tests/test_datasets/test_dataset.py | 39 +++++++++++++++++++++-------- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index e8d6e8778..60d65afc3 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -184,7 +184,7 @@ def decode_arff(fh): with io.open(filename, encoding='utf8') as fh: return decode_arff(fh) - def get_data(self, target=None, target_dtype=int, include_row_id=False, + def get_data(self, target=None, target_dtype=None, include_row_id=False, include_ignore_attributes=False, return_categorical_indicator=False, return_attribute_names=False): @@ -242,6 +242,12 @@ def get_data(self, target=None, target_dtype=int, include_row_id=False, else: if isinstance(target, six.string_types): target = [target] + legal_target_types = (int, float) + if target_dtype not in legal_target_types: + raise ValueError( + "%s is not a legal target type. Legal target types are %s" % + (target_dtype, legal_target_types) + ) targets = np.array([True if column in target else False for column in attribute_names]) diff --git a/openml/tasks/task.py b/openml/tasks/task.py index 127e7e232..9617c6e94 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -36,16 +36,20 @@ def get_dataset(self): return datasets.get_dataset(self.dataset_id) def get_X_and_y(self): + """Get data associated with the current task. + + Returns + ------- + tuple - X and y + + """ dataset = self.get_dataset() # Replace with retrieve from cache - if self.task_type_id == 1: - # if 'Supervised Classification'.lower() in self.task_type.lower(): + if self.task_type_id == 1: # Supervised classification target_dtype = int - # elif 'Supervised Regression'.lower() in self.task_type.lower(): - elif self.task_type_id == 2: + elif self.task_type_id == 2: # Supervised regression target_dtype = float - # elif ''.lower('Learning Curve') in self.task_type.lower(): - elif self.task_type_id == 3: + elif self.task_type_id == 3: # Learning curves task for classification target_dtype = int else: raise NotImplementedError(self.task_type) diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 0b11f3d73..5de5365f0 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -47,13 +47,16 @@ def test_get_data_with_rowid(self): self.assertEqual(len(categorical), 38) def test_get_data_with_target(self): - X, y = self.dataset.get_data(target="class") + X, y = self.dataset.get_data(target="class", target_dtype=int) self.assertIsInstance(X, np.ndarray) self.assertEqual(X.dtype, np.float32) self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (898, 38)) X, y, attribute_names = self.dataset.get_data( - target="class", return_attribute_names=True) + target="class", + target_dtype=int, + return_attribute_names=True + ) self.assertEqual(len(attribute_names), 38) self.assertNotIn("class", attribute_names) self.assertEqual(y.shape, (898, )) @@ -61,13 +64,20 @@ def test_get_data_with_target(self): def test_get_data_rowid_and_ignore_and_target(self): self.dataset.ignore_attributes = ["condition"] self.dataset.row_id_attribute = ["hardness"] - X, y = self.dataset.get_data(target="class", include_row_id=False, - include_ignore_attributes=False) + X, y = self.dataset.get_data( + target="class", + target_dtype=int, + include_row_id=False, + include_ignore_attributes=False + ) self.assertEqual(X.dtype, np.float32) self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (898, 36)) X, y, categorical = self.dataset.get_data( - target="class", return_categorical_indicator=True) + target="class", + target_dtype=int, + return_categorical_indicator=True, + ) self.assertEqual(len(categorical), 36) self.assertListEqual(categorical, [True] * 3 + [False] + [True] * 2 + [ False] + [True] * 23 + [False] * 3 + [True] * 3) @@ -100,14 +110,17 @@ def setUp(self): self.sparse_dataset = openml.datasets.get_dataset(4136) def test_get_sparse_dataset_with_target(self): - X, y = self.sparse_dataset.get_data(target="class") + X, y = self.sparse_dataset.get_data(target="class", target_dtype=int) self.assertTrue(sparse.issparse(X)) self.assertEqual(X.dtype, np.float32) self.assertIsInstance(y, np.ndarray) self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (600, 20000)) X, y, attribute_names = self.sparse_dataset.get_data( - target="class", return_attribute_names=True) + target="class", + target_dtype=int, + return_attribute_names=True, + ) self.assertTrue(sparse.issparse(X)) self.assertEqual(len(attribute_names), 20000) self.assertNotIn("class", attribute_names) @@ -170,14 +183,20 @@ def test_get_sparse_dataset_rowid_and_ignore_and_target(self): self.sparse_dataset.ignore_attributes = ["V256"] self.sparse_dataset.row_id_attribute = ["V512"] X, y = self.sparse_dataset.get_data( - target="class", include_row_id=False, - include_ignore_attributes=False) + target="class", + target_dtype=int, + include_row_id=False, + include_ignore_attributes=False, + ) self.assertTrue(sparse.issparse(X)) self.assertEqual(X.dtype, np.float32) self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (600, 19998)) X, y, categorical = self.sparse_dataset.get_data( - target="class", return_categorical_indicator=True) + target="class", + target_dtype=int, + return_categorical_indicator=True, + ) self.assertTrue(sparse.issparse(X)) self.assertEqual(len(categorical), 19998) self.assertListEqual(categorical, [False] * 19998) From 311c861cfaac16a47d0abc26f24b26a75119c6ad Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 12 Oct 2017 11:45:43 +0200 Subject: [PATCH 07/86] Simplify usage, make task primary object for beginners --- doc/usage.rst | 475 +++++++++++---------------------- examples/OpenML_Tutorial.ipynb | 18 +- 2 files changed, 162 insertions(+), 331 deletions(-) diff --git a/doc/usage.rst b/doc/usage.rst index 98453f4d0..f1d5c5181 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -31,7 +31,6 @@ programmatically after loading the package: .. code:: python >>> import openml - >>> apikey = 'Your API key' >>> openml.config.apikey = apikey @@ -64,191 +63,41 @@ Config file: cachedir = '~/.openml/cache' -~~~~~~~~~~~~~~~~~~~~~ -Working with datasets -~~~~~~~~~~~~~~~~~~~~~ - -# TODO mention third, searching for tags - -Datasets are a key concept in OpenML (see `OpenML documentation `_). -Datasets are identified by IDs and can be accessed in two different ways: - -1. In a list providing basic information on all datasets available on OpenML. - This function will not download the actual dataset, but will instead download - meta data which can be used to filter the datasets and retrieve a set of IDs. -2. A single dataset by its ID. A single dataset contains all meta information and the actual - data in form of an .arff file. The .arff file will be converted into a numpy - array by the OpenML Python API. - -Listing datasets -~~~~~~~~~~~~~~~~ - -A common task when using OpenML is to find a set of datasets which fulfill -several criteria. They should for example have between 1,000 and 10,000 -data points and at least five features. - -.. code:: python - - >>> datasets = openml.datasets.list_datasets() - -:meth:`openml.datasets.list_datasets` returns a dictionary of dictionaries, we -will convert it into a -`pandas dataframe `_ -to have better visualization and easier access: - -.. code:: python - - >>> import pandas as pd - >>> datasets = pd.DataFrame.from_dict(datasets, orient='index') - -We have access to the following properties of the datasets: - - >>> print(datasets.columns) - Index(['did', 'name', 'format', 'status', 'MajorityClassSize', - 'MaxNominalAttDistinctValues', 'MinorityClassSize', 'NumberOfClasses', - 'NumberOfFeatures', 'NumberOfInstances', - 'NumberOfInstancesWithMissingValues', 'NumberOfMissingValues', - 'NumberOfNumericFeatures', 'NumberOfSymbolicFeatures'], - dtype='object') - -and can see the first data point: - - >>> print(datasets.iloc[0]) - did 2 - name anneal - format ARFF - status active - MajorityClassSize 684 - MaxNominalAttDistinctValues 7 - MinorityClassSize 8 - NumberOfClasses 5 - NumberOfFeatures 39 - NumberOfInstances 898 - NumberOfInstancesWithMissingValues 898 - NumberOfMissingValues 22175 - NumberOfNumericFeatures 6 - NumberOfSymbolicFeatures 33 - Name: 2, dtype: object - -We can now filter the data: - - >>> filter = (datasets.NumberOfInstances > 1000) & (datasets.NumberOfFeatures > 5) - >>> filtered_datasets = datasets.loc[filter] - >>> dataset_indices = list(filtered_datasets.index) - >>> print(dataset_indices) # doctest: +SKIP - [3, 6, 12, 14, 16, 18, 20, 21, 22, 23, 24, 26, 28, 30, 32, 36, 38, 44, - ... 5291, 5293, 5295, 5296, 5297, 5301, 5587, 5648, 5889] - -and get a list of dataset indices which can be used in a next step. - -Downloading datasets -~~~~~~~~~~~~~~~~~~~~ - -We can now use the dataset IDs to download all datasets by their IDs. Let's -first look at how to download a single dataset and what can be done with the -dataset object: - -.. code:: python - - >>> dataset_id = 23 - >>> dataset = openml.datasets.get_dataset(dataset_id) - -Properties of the dataset are stored as member variables: - -.. code:: python - - >>> print(dataset.__dict__) # doctest: +SKIP - {'upload_date': u'2014-04-06 23:21:03', 'md5_cheksum': u'3149646ecff276abac3e892d1556655f', 'creator': None, 'citation': None, 'tag': [u'study_1', u'study_7', u'uci'], 'version_label': u'1', 'contributor': None, 'paper_url': None, 'original_data_url': None, 'id': 23, 'collection_date': None, 'row_id_attribute': None, 'version': 1, 'data_pickle_file': '/home/matthias/.openml/cache/datasets/23/dataset.pkl', 'default_target_attribute': u'Contraceptive_method_used', 'description': u"**Author**: \n**Source**: Unknown - \n**Please cite**: \n\n1. Title: Contraceptive Method Choice\n \n 2. Sources:\n (a) Origin: This dataset is a subset of the 1987 National Indonesia\n Contraceptive Prevalence Survey\n (b) Creator: Tjen-Sien Lim (limt@stat.wisc.edu)\n (c) Donor: Tjen-Sien Lim (limt@stat.wisc.edu)\n (c) Date: June 7, 1997\n \n 3. Past Usage:\n Lim, T.-S., Loh, W.-Y. & Shih, Y.-S. (1999). A Comparison of\n Prediction Accuracy, Complexity, and Training Time of Thirty-three\n Old and New Classification Algorithms. Machine Learning. Forthcoming.\n (ftp://ftp.stat.wisc.edu/pub/loh/treeprogs/quest1.7/mach1317.pdf or\n (http://www.stat.wisc.edu/~limt/mach1317.pdf)\n \n 4. Relevant Information:\n This dataset is a subset of the 1987 National Indonesia Contraceptive\n Prevalence Survey. The samples are married women who were either not \n pregnant or do not know if they were at the time of interview. The \n problem is to predict the current contraceptive method choice \n (no use, long-term methods, or short-term methods) of a woman based \n on her demographic and socio-economic characteristics.\n \n 5. Number of Instances: 1473\n \n 6. Number of Attributes: 10 (including the class attribute)\n \n 7. Attribute Information:\n \n 1. Wife's age (numerical)\n 2. Wife's education (categorical) 1=low, 2, 3, 4=high\n 3. Husband's education (categorical) 1=low, 2, 3, 4=high\n 4. Number of children ever born (numerical)\n 5. Wife's religion (binary) 0=Non-Islam, 1=Islam\n 6. Wife's now working? (binary) 0=Yes, 1=No\n 7. Husband's occupation (categorical) 1, 2, 3, 4\n 8. Standard-of-living index (categorical) 1=low, 2, 3, 4=high\n 9. Media exposure (binary) 0=Good, 1=Not good\n 10. Contraceptive method used (class attribute) 1=No-use \n 2=Long-term\n 3=Short-term\n \n 8. Missing Attribute Values: None\n\n Information about the dataset\n CLASSTYPE: nominal\n CLASSINDEX: last", 'format': u'ARFF', 'visibility': u'public', 'update_comment': None, 'licence': u'Public', 'name': u'cmc', 'language': None, 'url': u'http://www.openml.org/data/download/23/dataset_23_cmc.arff', 'data_file': '~/.openml/cache/datasets/23/dataset.arff', 'ignore_attributes': None} - -Next, to obtain the data matrix: - -.. code:: python - - >>> X = dataset.get_data() - >>> print(X.shape, X.dtype) - (1473, 10) float32 - -which returns the dataset as a np.ndarray with dtype :python:`np.float32`. -In case the data is sparse, a scipy.sparse.csr matrix is returned. All nominal -variables are encoded as integers, the inverse encoding can be retrieved via: - -.. code:: python - - >>> X, names = dataset.get_data(return_attribute_names=True) - >>> print(names) - ['Wifes_age', 'Wifes_education', 'Husbands_education', 'Number_of_children_ever_born', 'Wifes_religion', 'Wifes_now_working%3F', 'Husbands_occupation', 'Standard-of-living_index', 'Media_exposure', 'Contraceptive_method_used'] - -Most times, having a single data matrix :python:`X` is not enough. Two -useful arguments are :python:`target` and -:python:`return_categorical_indicator`. :python:`target` makes -:meth:`get_data()` return :python:`X` and :python:`y` -seperate; :python:`return_categorical_indicator` makes -:meth:`get_data()` return a boolean array which indicate -which attributes are categorical (and should be one hot encoded if necessary.) - -.. code:: python - - >>> X, y, categorical = dataset.get_data( - ... target=dataset.default_target_attribute, - ... return_categorical_indicator=True) - >>> print(X.shape, y.shape) - (1473, 9) (1473,) - >>> print(categorical) - [False, True, True, False, True, True, True, True, True] -In case you are working with `scikit-learn -`_, you can use this data right away: +~~~~~~~~~~~~ +Key concepts +~~~~~~~~~~~~ -.. code:: python - - >>> from sklearn import preprocessing, ensemble - >>> enc = preprocessing.OneHotEncoder(categorical_features=categorical) - >>> print(enc) - OneHotEncoder(categorical_features=[False, True, True, False, True, True, True, True, True], - dtype=, handle_unknown='error', - n_values='auto', sparse=True) - >>> X = enc.fit_transform(X).todense() - >>> clf = ensemble.RandomForestClassifier() - >>> clf.fit(X, y) - RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini', - max_depth=None, max_features='auto', max_leaf_nodes=None, - min_impurity_split=1e-07, min_samples_leaf=1, - min_samples_split=2, min_weight_fraction_leaf=0.0, - n_estimators=10, n_jobs=1, oob_score=False, random_state=None, - verbose=0, warm_start=False) - -When you have to retrieve several datasets, you can use the convenience function -:meth:`openml.datasets.get_datasets()`, which downloads all datasets given by -a list of IDs: - - >>> ids = [12, 14, 16, 18, 20, 22] - >>> datasets = openml.datasets.get_datasets(ids) - >>> print(datasets[0].name) - mfeat-factors +OpenML contains several key concepts which it needs to make machine learning +research shareable. A machine learning experiment consists of several runs, +which describe the performance of an algorithm (called a flow in OpenML) on a +task. Task is the combination of a dataset, a split and an evaluation metric. In +this user guide we will go through listing and exploring existing tasks to +actually running machine learning algorithms on them. In a further user guide +we will examine how to search through datasets in order to curate a list of +tasks. ~~~~~~~~~~~~~~~~~~ Working with tasks ~~~~~~~~~~~~~~~~~~ -#TODO put a link to the OpenML documentation here! Link the Task functions and -the task class - -While datasets provide the most basic information for a machine learning task, -they do not provide enough information for a reproducible machine learning -experiment. A task defines how to split the dataset into a train and test set, -whether to use several disjoint train and test splits (cross-validation) and -whether this should be repeated several times. Also, the task defines a target -metric for which a flow should be optimized. +Tasks are containers, defining how to split the dataset into a train and test +set, whether to use several disjoint train and test splits (cross-validation) +and whether this should be repeated several times. Also, the task defines a +target metric for which a flow should be optimized. You can think of a task as +an experimentation protocol, describing how to apply a machine learning model +to a dataset in a way that it is comparable with the results of others (more +on how to do that further down). -Just like datasets, tasks are identified by IDs and can be accessed in three -different ways: +Tasks are identified by IDs and can be accessed in two different ways: 1. In a list providing basic information on all tasks available on OpenML. This function will not download the actual tasks, but will instead download meta data that can be used to filter the tasks and retrieve a set of IDs. -2. By functions only list a subset of all available tasks, restricted either by - their :TODO:`task_type`, :TODO:`tag` or :TODO:`check_for_more`. -3. A single task by its ID. It contains all meta information, the target metric, + We can filter this list, for example, we can only list + *supervised classification* tasks or tasks having a special tag. + +2. A single task by its ID. It contains all meta information, the target metric, the splits and an iterator which can be used to access the splits in a useful manner. @@ -257,25 +106,17 @@ You can also read more about tasks in the `OpenML guide >> tasks = openml.tasks.list_tasks(task_type_id=1) -Let's find out more about the datasets: +:meth:`openml.tasks.list_tasks` returns a dictionary of dictionaries, we convert +it into a +`pandas dataframe `_ +to have better visualization and easier access: .. code:: python @@ -293,56 +134,40 @@ Let's find out more about the datasets: Now we can restrict the tasks to all tasks with the desired resampling strategy: -# TODO add something about the different resampling strategies implemented! - .. code:: python - >>> filtered_tasks = tasks.query('estimation_procedure == "10-fold Crossvalidation"') - >>> filtered_tasks = list(filtered_tasks.index) - >>> print(filtered_tasks) # doctest: +SKIP - [1, 2, 3, 4, 5, 6, 7, 8, 9, ... 10105, 10106, 10107, 10109, 10111, 13907, 13918] -Resampling strategies can be found on the `OpenML Website `_ -or programatically as described in `Finding out evaluation strategies and target metrics`_. - -Finally, we can check whether there is a task for each dataset that we want to -use in our study. If this is not the case, tasks can be created on the -`OpenML website `_. + >>> filtered_tasks = tasks.query('estimation_procedure == "10-fold Crossvalidation"') + >>> print(list(filtered_tasks.index)) # doctest: +SKIP + [2, 3, 4, 5, 6, 7, 8, 9, ..., 146606, 146607, 146690] + >>> print(len(filtered_tasks)) # doctest: +SKIP + 1697 -The rest of this subsection deals with accessing a list of tasks by tags and -without any restriction. +Resampling strategies can be found on the `OpenML Website `_. -A list of tasks, filtered tags, can be retrieved via: +We can further filter the list of tasks to only contain datasets with more than +500 samples, but less than 1000 samples: .. code:: python - >>> tasks = openml.tasks.list_tasks(tag='study_1') + >>> filtered_tasks = filtered_tasks.query('NumberOfInstances > 500 and NumberOfInstances < 1000') + >>> print(list(filtered_tasks.index)) # doctest: +SKIP + [2, 11, 15, 29, 37, 41, 49, 53, ..., 146231, 146238, 146241] + >>> print(len(filtered_tasks)) + 107 -:meth:`openml.tasks.list_tasks` returns a dict of dictionaries, we will -convert it into a `pandas dataframe `_ -to have better visualization: +Similar to listing tasks by task type, we can list tasks by tags: .. code:: python - >>> import pandas as pd + >>> tasks = openml.tasks.list_tasks(tag='OpenML100') >>> tasks = pd.DataFrame.from_dict(tasks, orient='index') -As before, we have to check whether there is a task for each dataset that we -want to work with. In addition, we have to make sure to use only tasks with the -desired task type: - -#TODO this doesn't look nice, we should have a constant for each known task, -dynamically created by the task type available (but when do we know that we -can savely use the api connector? what to do if we do not have an internet -connection? Maybe have this statically in the program and check from time to -time if there is something new (via a unit test?)?, the same holds true for -the resampling strategies available!) - -.. code:: python - - >>> filter = tasks.task_type == 'Supervised Classification' - >>> filtered_tasks = tasks[filter] - >>> print(len(filtered_tasks)) # doctest: +SKIP - 2599 +*OpenML 100* is a curated list of 100 tasks to start using OpenML. They are all +supervised classification tasks with more than 500 instances and less than 50000 +instances per task. To make things easier, the tasks do not contain highly +unbalanced data and sparse data. However, the tasks include missing values and +categorical features. You can find out more about the *OpenML 100* on +`the OpenML benchmarking page `_. Finally, it is also possible to list all tasks on OpenML with: @@ -350,14 +175,14 @@ Finally, it is also possible to list all tasks on OpenML with: >>> tasks = openml.tasks.list_tasks() >>> print(len(tasks)) # doctest: +SKIP - 29757 + 46067 Downloading tasks ~~~~~~~~~~~~~~~~~ -Downloading tasks works similar to downloading datasets. We provide two -functions for this, one which downloads only a single task by its ID, -and one which takes a list of IDs and downloads all of these tasks: +We provide two functions to download tasks, one which downloads only a single +task by its ID, and one which takes a list of IDs and downloads all of these +tasks: .. code:: python @@ -390,7 +215,7 @@ Properties of the task are stored as member variables: 'task_type': 'Supervised Classification', 'task_type_id': 1} -And with a list of task IDs: +And: .. code:: python @@ -398,81 +223,14 @@ And with a list of task IDs: >>> tasks = openml.tasks.get_tasks(ids) >>> pprint(tasks[0]) # doctest: +SKIP -~~~~~~~~~~~~~~~~~~~~~~~ -Finding out tasks types -~~~~~~~~~~~~~~~~~~~~~~~ - -Not yet supported by the API. Please use the OpenML website. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Finding out evaluation strategies and target metrics -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Not yet supported by the API. Please use the OpenML website. - -~~~~~~~~~~~~~~~ -Using the cache -~~~~~~~~~~~~~~~ - -Downloading all datasets, tasks and split every time a get function is called -would prohibit a user to interact with the API in an exploratory manner. -OpenML is designed in a way that certain entities are immutable once created. -This allows the python package to cache datasets, tasks, splits and runs locally -for fast retrieval. Another benefit is that the API can be used normally on a -compute cluster without internet access (:ref:`see below`). - -Currently, the following objects are cached: - -* datasets - * dataset arff. In order to reduce parsing time, the data is serialized to - disk in a binary format (using the `pickle library `_. - * dataset descriptions - * more? -* tasks - * task description - * split arff. TODO are they cached? -* runs - * run description - -Run predictions are not cached yet. Flow ojects cannot yet be downloaded and are -therefore not cached. - -Configuring the cache -~~~~~~~~~~~~~~~~~~~~~ - -Configuring the cache works as described in the subsection `Connecting to the OpenML server`_: -It can be done either through the API: - -.. code:: python - - >>> openml.config.set_cache_directory(os.path.expanduser('~/.openml/cache')) - -or the config file: - -.. code:: bash - - cachedir = '~/.openml/cache' - - -Clearing the cache -~~~~~~~~~~~~~~~~~~ - -Currently, there is no programmatic way to interact with the cache and we do not -plan to implement one. If you have any use case for this, please open an issue -on the `issue tracker `_. - -# TODO check that the cache is in a consistent state! -In case the cache gets too large, you can manually delete unnecessary files. -Make sure that you always delete a complete entity, for example the whole -directory caching a dataset named after the datasets ID. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Working with Flows and Runs -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~ +Creating runs +~~~~~~~~~~~~~ -Tasks and datasets allow us to download all information to run an experiment -locally. In order to upload and share results of such an experiment we need -the concepts of flows and runs. +In order to upload and share results of running a machine learning algorithm +on a task, we need to create an :class:`~openml.OpenMLRun`. A run object can +be created by running a :class:`~openml.OpenMLFlow` or a scikit-learn model on +a task. We will focus on the simpler example of running a scikit-learn model. Flows are descriptions of something runable which does the machine learning. A flow contains all information to set up the necessary machine learning @@ -499,6 +257,80 @@ Running a model >>> task = openml.tasks.get_task(12) >>> run = openml.runs.run_model_on_task(task, model) >>> pprint(vars(run), depth=2) # doctest: +SKIP + {'data_content': [...], + 'dataset_id': 12, + 'error_message': None, + 'evaluations': None, + 'flow': None, + 'flow_id': 7257, + 'flow_name': None, + 'fold_evaluations': defaultdict(. at 0x7fb88981b9d8>, + {'predictive_accuracy': defaultdict(, + {0: {0: 0.94499999999999995, + 1: 0.94499999999999995, + 2: 0.94499999999999995, + 3: 0.96499999999999997, + 4: 0.92500000000000004, + 5: 0.96499999999999997, + 6: 0.94999999999999996, + 7: 0.96999999999999997, + 8: 0.93999999999999995, + 9: 0.95499999999999996}}), + 'usercpu_time_millis': defaultdict(, + {0: {0: 110.4880920000042, + 1: 105.7469440000034, + 2: 107.4153629999941, + 3: 105.1104170000059, + 4: 104.02388900000403, + 5: 105.17172800000196, + 6: 109.00792000001047, + 7: 107.49670599999206, + 8: 107.34138000000115, + 9: 104.78881499999915}}), + 'usercpu_time_millis_testing': defaultdict(, + {0: {0: 3.6470320000034917, + 1: 3.5307810000020368, + 2: 3.5432540000002177, + 3: 3.5460690000022055, + 4: 3.5634600000022942, + 5: 3.906016000001955, + 6: 3.6680000000046675, + 7: 3.643865999997331, + 8: 3.4515420000005292, + 9: 3.461469000001216}}), + 'usercpu_time_millis_training': defaultdict(, + {0: {0: 106.84106000000071, + 1: 102.21616300000136, + 2: 103.87210899999388, + 3: 101.56434800000369, + 4: 100.46042900000174, + 5: 101.26571200000001, + 6: 105.3399200000058, + 7: 103.85283999999473, + 8: 103.88983800000062, + 9: 101.32734599999793}})}), + 'model': RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini', + max_depth=None, max_features='auto', max_leaf_nodes=None, + min_impurity_split=1e-07, min_samples_leaf=1, + min_samples_split=2, min_weight_fraction_leaf=0.0, + n_estimators=10, n_jobs=1, oob_score=False, random_state=43934, + verbose=0, warm_start=False), + 'output_files': None, + 'parameter_settings': [...], + 'predictions_url': None, + 'run_id': None, + 'sample_evaluations': None, + 'setup_id': None, + 'setup_string': None, + 'tags': [...], + 'task': None, + 'task_evaluation_measure': None, + 'task_id': 12, + 'task_type': None, + 'trace_attributes': None, + 'trace_content': None, + 'uploader': None, + 'uploader_name': None} So far the run is only available locally. By calling the publish function, the run is send to the OpenML server: @@ -506,14 +338,14 @@ run is send to the OpenML server: .. code:: python >>> run.publish() # doctest: +SKIP - # What happens here? What should it return? + We can now also inspect the flow object which was automatically created: .. code:: python >>> flow = openml.flows.get_flow(run.flow_id) - >>> pprint(vars(flow), depth=2) # doctest: +SKIP + >>> pprint(vars(flow), depth=1) # doctest: +SKIP {'binary_format': None, 'binary_md5': None, 'binary_url': None, @@ -523,7 +355,7 @@ We can now also inspect the flow object which was automatically created: 'dependencies': 'sklearn==0.18.2\nnumpy>=1.6.1\nscipy>=0.9', 'description': 'Automatically created scikit-learn flow.', 'external_version': 'openml==0.6.0,sklearn==0.18.2', - 'flow_id': 7245, + 'flow_id': 7257, 'language': 'English', 'model': RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini', max_depth=None, max_features='auto', max_leaf_nodes=None, @@ -534,23 +366,18 @@ We can now also inspect the flow object which was automatically created: 'name': 'sklearn.ensemble.forest.RandomForestClassifier', 'parameters': OrderedDict([...]), 'parameters_meta_info': OrderedDict([...]), - 'tags': ['openml-python', - 'python', - 'scikit-learn', - 'sklearn', - 'sklearn_0.18.2'], - 'upload_date': '2017-10-06T14:54:38', - 'uploader': '86', - 'version': '28'} - -Retrieving results from OpenML -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# TODO - - - - + 'tags': [...], + 'upload_date': '2017-10-09T10:20:40', + 'uploader': '1159', + 'version': '29'} +Advanced topics +~~~~~~~~~~~~~~~ +We are working on tutorials for the following topics: +* Querying datasets +* Uploading datasets +* Creating tasks +* Working offline +* Analyzing large amounts of results diff --git a/examples/OpenML_Tutorial.ipynb b/examples/OpenML_Tutorial.ipynb index dcc7aedec..2632bc2ed 100644 --- a/examples/OpenML_Tutorial.ipynb +++ b/examples/OpenML_Tutorial.ipynb @@ -24,9 +24,7 @@ }, { "cell_type": "raw", - "metadata": { - "collapsed": true - }, + "metadata": {}, "source": [ "# Install OpenML (developer version)\n", "# 'pip install openml' coming up (october 2017) \n", @@ -842,8 +840,10 @@ ], "source": [ "X, y, attribute_names = dataset.get_data(\n", - " target=dataset.default_target_attribute, \n", - " return_attribute_names=True)\n", + " target=dataset.default_target_attribute,\n", + " target_dtype=int,\n", + " return_attribute_names=True,\n", + ")\n", "eeg = pd.DataFrame(X, columns=attribute_names)\n", "eeg['class'] = y\n", "print(eeg[:10])" @@ -932,7 +932,10 @@ "from sklearn import neighbors\n", "\n", "dataset = oml.datasets.get_dataset(1471)\n", - "X, y = dataset.get_data(target=dataset.default_target_attribute)\n", + "X, y = dataset.get_data(\n", + " target=dataset.default_target_attribute,\n", + " target_dtype=int,\n", + ")\n", "clf = neighbors.KNeighborsClassifier(n_neighbors=1)\n", "clf.fit(X, y)" ] @@ -989,6 +992,7 @@ "dataset = oml.datasets.get_dataset(10)\n", "X, y, categorical = dataset.get_data(\n", " target=dataset.default_target_attribute,\n", + " target_dtype=int,\n", " return_categorical_indicator=True)\n", "print(\"Categorical features: %s\" % categorical)\n", "enc = preprocessing.OneHotEncoder(categorical_features=categorical)\n", @@ -1547,7 +1551,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.0" + "version": "3.6.1" } }, "nbformat": 4, From aa758f9ab6e0608ede4c4199d88fd533c6eb2ad1 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 12 Oct 2017 14:12:59 +0200 Subject: [PATCH 08/86] FIX cast data qualities to float --- openml/datasets/dataset.py | 25 ++++++++++++++----- openml/datasets/functions.py | 4 +-- tests/test_datasets/test_dataset.py | 15 +++++++++++ tests/test_datasets/test_dataset_functions.py | 4 +-- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index e8d6e8778..6e4116ebc 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -88,12 +88,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None, raise ValueError('Data features not provided in right order') self.features[feature.index] = feature - if qualities is not None: - self.qualities = {} - for idx, xmlquality in enumerate(qualities['oml:quality']): - name = xmlquality['oml:name'] - value = xmlquality['oml:value'] - self.qualities[name] = value + self.qualities = _check_qualities(qualities) if data_file is not None: if self._data_features_supported(): @@ -426,3 +421,21 @@ def _data_features_supported(self): return False return True return True + + + +def _check_qualities(qualities): + if qualities is not None: + qualities_ = {} + for xmlquality in qualities: + name = xmlquality['oml:name'] + if xmlquality['oml:value'] is None: + value = float('NaN') + elif xmlquality['oml:value'] == 'null': + value = float('NaN') + else: + value = float(xmlquality['oml:value']) + qualities_[name] = value + return qualities_ + else: + return None diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 078dc3faa..dd7bcb359 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -116,7 +116,7 @@ def _get_cached_dataset_qualities(dataset_id): try: with io.open(qualities_file, encoding='utf8') as fh: qualities_xml = fh.read() - return xmltodict.parse(qualities_xml)["oml:data_qualities"] + return xmltodict.parse(qualities_xml)["oml:data_qualities"]['oml:quality'] except (IOError, OSError): raise OpenMLCacheException("Dataset qualities for dataset id %d not " "cached" % dataset_id) @@ -452,7 +452,7 @@ def _get_dataset_qualities(did_cache_dir, dataset_id): with io.open(qualities_file, "w", encoding='utf8') as fh: fh.write(qualities_xml) - qualities = xmltodict.parse(qualities_xml, force_list=('oml:quality',))['oml:data_qualities'] + qualities = xmltodict.parse(qualities_xml, force_list=('oml:quality',))['oml:data_qualities']['oml:quality'] return qualities diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 0b11f3d73..75f4b0355 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -182,3 +182,18 @@ def test_get_sparse_dataset_rowid_and_ignore_and_target(self): self.assertEqual(len(categorical), 19998) self.assertListEqual(categorical, [False] * 19998) self.assertEqual(y.shape, (600, )) + + +class OpenMLDatasetQualityTest(TestBase): + def test__check_qualities(self): + qualities = [{'oml:name': 'a', 'oml:value': '0.5'}] + qualities = openml.datasets.dataset._check_qualities(qualities) + self.assertEqual(qualities['a'], 0.5) + + qualities = [{'oml:name': 'a', 'oml:value': 'null'}] + qualities = openml.datasets.dataset._check_qualities(qualities) + self.assertNotEqual(qualities['a'], qualities['a']) + + qualities = [{'oml:name': 'a', 'oml:value': None}] + qualities = openml.datasets.dataset._check_qualities(qualities) + self.assertNotEqual(qualities['a'], qualities['a']) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 2a0d6be83..1623f2006 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -74,7 +74,7 @@ def test__get_cached_dataset(self, ): self.assertIsInstance(dataset, OpenMLDataset) self.assertTrue(len(dataset.features) > 0) self.assertTrue(len(dataset.features) == len(features['oml:feature'])) - self.assertTrue(len(dataset.qualities) == len(qualities['oml:quality'])) + self.assertTrue(len(dataset.qualities) == len(qualities)) def test_get_cached_dataset_description(self): openml.config.set_cache_directory(self.static_cache_dir) @@ -210,7 +210,7 @@ def test__get_dataset_features(self): def test__get_dataset_qualities(self): # Only a smoke check qualities = _get_dataset_qualities(self.workdir, 2) - self.assertIsInstance(qualities, dict) + self.assertIsInstance(qualities, list) def test_deletion_of_cache_dir(self): # Simple removal From 4181c4a91b77435df8e101f012c722cdd909b8d2 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 12 Oct 2017 17:22:19 +0200 Subject: [PATCH 09/86] include suggestions from @amueller --- doc/usage.rst | 65 ++++++++++++++++++++------------------ openml/datasets/dataset.py | 2 +- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/doc/usage.rst b/doc/usage.rst index f1d5c5181..61a223af4 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -69,33 +69,36 @@ Key concepts ~~~~~~~~~~~~ OpenML contains several key concepts which it needs to make machine learning -research shareable. A machine learning experiment consists of several runs, -which describe the performance of an algorithm (called a flow in OpenML) on a -task. Task is the combination of a dataset, a split and an evaluation metric. In -this user guide we will go through listing and exploring existing tasks to -actually running machine learning algorithms on them. In a further user guide -we will examine how to search through datasets in order to curate a list of -tasks. +research shareable. A machine learning experiment consists of one or several +**runs**, which describe the performance of an algorithm (called a **flow** in +OpenML), its hyperparameter settings (called a **setup**) on a **task**. A +**Task** is the combination of a **dataset**, a split and an evaluation +metric. In this user guide we will go through listing and exploring existing +**tasks** to actually running machine learning algorithms on them. In a further +user guide we will examine how to search through **datasets** in order to curate +a list of **tasks**. ~~~~~~~~~~~~~~~~~~ Working with tasks ~~~~~~~~~~~~~~~~~~ -Tasks are containers, defining how to split the dataset into a train and test -set, whether to use several disjoint train and test splits (cross-validation) -and whether this should be repeated several times. Also, the task defines a -target metric for which a flow should be optimized. You can think of a task as -an experimentation protocol, describing how to apply a machine learning model -to a dataset in a way that it is comparable with the results of others (more -on how to do that further down). +You can think of a task as an experimentation protocol, describing how to apply +a machine learning model to a dataset in a way that it is comparable with the +results of others (more on how to do that further down).Tasks are containers, +defining which dataset to use, what kind of task we're solving (regression, +classification, clustering, etc...) and which column to predict. Furthermore, +it also describes how to split the dataset into a train and test set, whether +to use several disjoint train and test splits (cross-validation) and whether +this should be repeated several times. Also, the task defines a target metric +for which a flow should be optimized. Tasks are identified by IDs and can be accessed in two different ways: 1. In a list providing basic information on all tasks available on OpenML. This function will not download the actual tasks, but will instead download meta data that can be used to filter the tasks and retrieve a set of IDs. - We can filter this list, for example, we can only list - *supervised classification* tasks or tasks having a special tag. + We can filter this list, for example, we can only list tasks having a special + tag or only tasks for a specific target such as *supervised classification*. 2. A single task by its ID. It contains all meta information, the target metric, the splits and an iterator which can be used to access the splits in a @@ -132,29 +135,30 @@ to have better visualization and easier access: 'NumberOfSymbolicFeatures', 'cost_matrix'], dtype='object') -Now we can restrict the tasks to all tasks with the desired resampling strategy: +We can filter the list of tasks to only contain datasets with more than +500 samples, but less than 1000 samples: .. code:: python - >>> filtered_tasks = tasks.query('estimation_procedure == "10-fold Crossvalidation"') + >>> filtered_tasks = tasks.query('NumberOfInstances > 500 and NumberOfInstances < 1000') >>> print(list(filtered_tasks.index)) # doctest: +SKIP - [2, 3, 4, 5, 6, 7, 8, 9, ..., 146606, 146607, 146690] - >>> print(len(filtered_tasks)) # doctest: +SKIP - 1697 - -Resampling strategies can be found on the `OpenML Website `_. + [2, 11, 15, 29, 37, 41, 49, 53, ..., 146597, 146600, 146605] + >>> print(len(filtered_tasks)) + 210 -We can further filter the list of tasks to only contain datasets with more than -500 samples, but less than 1000 samples: +Then, we can further restrict the tasks to all have the same resampling +strategy: .. code:: python - >>> filtered_tasks = filtered_tasks.query('NumberOfInstances > 500 and NumberOfInstances < 1000') + >>> filtered_tasks = filtered_tasks.query('estimation_procedure == "10-fold Crossvalidation"') >>> print(list(filtered_tasks.index)) # doctest: +SKIP [2, 11, 15, 29, 37, 41, 49, 53, ..., 146231, 146238, 146241] - >>> print(len(filtered_tasks)) + >>> print(len(filtered_tasks)) # doctest: +SKIP 107 +Resampling strategies can be found on the `OpenML Website `_. + Similar to listing tasks by task type, we can list tasks by tags: .. code:: python @@ -219,7 +223,7 @@ And: .. code:: python - >>> ids = [12, 14, 16, 18, 20, 22] + >>> ids = [2, 11, 15, 29, 37, 41, 49, 53] >>> tasks = openml.tasks.get_tasks(ids) >>> pprint(tasks[0]) # doctest: +SKIP @@ -229,8 +233,9 @@ Creating runs In order to upload and share results of running a machine learning algorithm on a task, we need to create an :class:`~openml.OpenMLRun`. A run object can -be created by running a :class:`~openml.OpenMLFlow` or a scikit-learn model on -a task. We will focus on the simpler example of running a scikit-learn model. +be created by running a :class:`~openml.OpenMLFlow` or a scikit-learn compatible +model on a task. We will focus on the simpler example of running a +scikit-learn model. Flows are descriptions of something runable which does the machine learning. A flow contains all information to set up the necessary machine learning diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 60d65afc3..a5b88c38b 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -242,7 +242,7 @@ def get_data(self, target=None, target_dtype=None, include_row_id=False, else: if isinstance(target, six.string_types): target = [target] - legal_target_types = (int, float) + legal_target_types = (int, float, np.float32, np.float64) if target_dtype not in legal_target_types: raise ValueError( "%s is not a legal target type. Legal target types are %s" % From 90fab5387d5591256792a7208395e767205f5e42 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Thu, 12 Oct 2017 17:36:38 +0200 Subject: [PATCH 10/86] add dataset tagging, make search return empty list, not exception --- openml/_api_calls.py | 5 ++- openml/datasets/dataset.py | 32 ++++++++++++++----- openml/datasets/functions.py | 7 ++-- openml/exceptions.py | 7 +++- tests/test_datasets/test_dataset.py | 19 +++++++++++ tests/test_datasets/test_dataset_functions.py | 1 - 6 files changed, 58 insertions(+), 13 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 043759559..7fa2efefb 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -7,7 +7,8 @@ import xmltodict from . import config -from .exceptions import OpenMLServerError, OpenMLServerException +from .exceptions import (OpenMLServerError, OpenMLServerException, + OpenMLServerNoResult) def _perform_api_call(call, data=None, file_dictionary=None, @@ -138,4 +139,6 @@ def _parse_server_exception(response): additional = None if 'oml:additional_information' in server_exception['oml:error']: additional = server_exception['oml:error']['oml:additional_information'] + if code in [370, 372]: + return OpenMLServerNoResult(code, message, additional) return OpenMLServerException(code, message, additional) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index e8d6e8778..5b489b49b 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -3,7 +3,6 @@ import logging import os import six -import sys import arff @@ -82,7 +81,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None, feature = OpenMLDataFeature(int(xmlfeature['oml:index']), xmlfeature['oml:name'], xmlfeature['oml:data_type'], - None, #todo add nominal values (currently not in database) + None, # todo add nominal values (currently not in database) int(xmlfeature.get('oml:number_of_missing_values', 0))) if idx != feature.index: raise ValueError('Data features not provided in right order') @@ -129,6 +128,28 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None, logger.debug("Saved dataset %d: %s to file %s" % (self.dataset_id, self.name, self.data_pickle_file)) + def push_tag(self, tag): + """Annotates this data set with a tag on the server. + + Parameters + ---------- + tag : string + Tag to attach to the dataset. + """ + data = {'data_id': self.dataset_id, 'tag': tag} + _perform_api_call("/data/tag", data=data) + + def remove_tag(self, tag): + """Removes a tag from this dataset on the server. + + Parameters + ---------- + tag : string + Tag to attach to the dataset. + """ + data = {'data_id': self.dataset_id, 'tag': tag} + _perform_api_call("/data/untag", data=data) + def __eq__(self, other): if type(other) != OpenMLDataset: return False @@ -315,7 +336,6 @@ def retrieve_class_labels(self, target_name='class'): else: return None - def get_features_by_type(self, data_type, exclude=None, exclude_ignore_attributes=True, exclude_row_id_attribute=True): @@ -377,11 +397,7 @@ def publish(self): Returns ------- - return_code : int - Return code from server - - return_value : string - xml return from server + self """ file_elements = {'description': self._to_xml()} diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 478e19176..5c3243931 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -8,7 +8,7 @@ import xmltodict from .dataset import OpenMLDataset -from ..exceptions import OpenMLCacheException +from ..exceptions import OpenMLCacheException, OpenMLServerNoResult from .. import config from .._api_calls import _perform_api_call, _read_url @@ -178,7 +178,10 @@ def list_datasets(offset=None, size=None, tag=None): def _list_datasets(api_call): # TODO add proper error handling here! - xml_string = _perform_api_call(api_call) + try: + xml_string = _perform_api_call(api_call) + except OpenMLServerNoResult: + return [] datasets_dict = xmltodict.parse(xml_string, force_list=('oml:dataset',)) # Minimalistic check if the XML is useful diff --git a/openml/exceptions.py b/openml/exceptions.py index ae6f6be32..eb5890a1c 100644 --- a/openml/exceptions.py +++ b/openml/exceptions.py @@ -11,7 +11,7 @@ class OpenMLServerError(PyOpenMLError): def __init__(self, message): super(OpenMLServerError, self).__init__(message) -# + class OpenMLServerException(OpenMLServerError): """exception for when the result of the server was not 200 (e.g., listing call w/o results). """ @@ -22,6 +22,11 @@ def __init__(self, code, message, additional=None): super(OpenMLServerException, self).__init__(message) +class OpenMLServerNoResult(OpenMLServerException): + """exception for when the result of the server is empty. """ + pass + + class OpenMLCacheException(PyOpenMLError): """Dataset / task etc not found in cache""" def __init__(self, message): diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 0b11f3d73..3e3cf4a2b 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -90,6 +90,25 @@ def test_get_data_with_ignore_attributes(self): # TODO test multiple ignore attributes! +class OpenMLDatasetTestOnTestServer(TestBase): + def setUp(self): + super(OpenMLDatasetTestOnTestServer, self).setUp() + # longley, really small dataset + self.dataset = openml.datasets.get_dataset(125) + + def test_tagging(self): + tag = "testing_tag{}".format(self.id) + ds_list = openml.datasets.list_datasets(tag=tag) + self.assertEqual(len(ds_list), 0) + self.dataset.push_tag(tag) + ds_list = openml.datasets.list_datasets(tag=tag) + self.assertEqual(len(ds_list), 1) + self.assertEqual(ds_list[0]['did'], 125) + self.dataset.remove_tag(tag) + ds_list = openml.datasets.list_datasets(tag=tag) + self.assertEqual(len(ds_list), 0) + + class OpenMLDatasetTestSparse(TestBase): _multiprocess_can_split_ = True diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 2a0d6be83..d58ffff6c 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -1,6 +1,5 @@ import unittest import os -import os import sys if sys.version_info[0] >= 3: From 21e000764e0fbceef2a2d9fce0e1ab607e2da14d Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Thu, 12 Oct 2017 17:44:31 +0200 Subject: [PATCH 11/86] fix test for dataset tagging --- tests/test_datasets/test_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 3e3cf4a2b..5654b7e24 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -1,6 +1,7 @@ import numpy as np from scipy import sparse import six +from time import time from openml.testing import TestBase import openml @@ -97,13 +98,13 @@ def setUp(self): self.dataset = openml.datasets.get_dataset(125) def test_tagging(self): - tag = "testing_tag{}".format(self.id) + tag = "testing_tag_{}_{}".format(self.id(), time()) ds_list = openml.datasets.list_datasets(tag=tag) self.assertEqual(len(ds_list), 0) self.dataset.push_tag(tag) ds_list = openml.datasets.list_datasets(tag=tag) self.assertEqual(len(ds_list), 1) - self.assertEqual(ds_list[0]['did'], 125) + self.assertIn(125, ds_list) self.dataset.remove_tag(tag) ds_list = openml.datasets.list_datasets(tag=tag) self.assertEqual(len(ds_list), 0) From 96a850bdab4822f4182e75504c30b1bc78463f21 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Thu, 12 Oct 2017 18:00:26 +0200 Subject: [PATCH 12/86] use str instead of string as type --- openml/datasets/dataset.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 5b489b49b..28ab37f90 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -25,9 +25,9 @@ class OpenMLDataset(object): Parameters ---------- - name : string + name : str Name of the dataset - description : string + description : str Description of the dataset FIXME : which of these do we actually nee? """ @@ -133,7 +133,7 @@ def push_tag(self, tag): Parameters ---------- - tag : string + tag : str Tag to attach to the dataset. """ data = {'data_id': self.dataset_id, 'tag': tag} @@ -144,7 +144,7 @@ def remove_tag(self, tag): Parameters ---------- - tag : string + tag : str Tag to attach to the dataset. """ data = {'data_id': self.dataset_id, 'tag': tag} @@ -417,7 +417,7 @@ def _to_xml(self): Returns ------- - xml_dataset : string + xml_dataset : str XML description of the data. """ xml_dataset = (' Date: Thu, 12 Oct 2017 18:15:39 +0200 Subject: [PATCH 13/86] add tagging for runs, don't error on empty list_runs --- openml/_api_calls.py | 3 ++- openml/runs/functions.py | 8 ++++--- openml/runs/run.py | 25 +++++++++++++++++++- tests/test_runs/test_run.py | 46 ++++++++++++++++++++++++------------- 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 7fa2efefb..2fee9098e 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -139,6 +139,7 @@ def _parse_server_exception(response): additional = None if 'oml:additional_information' in server_exception['oml:error']: additional = server_exception['oml:error']['oml:additional_information'] - if code in [370, 372]: + if code in [370, 372, 512]: + # 512 for runs, 370 for datasets (should be 372) return OpenMLServerNoResult(code, message, additional) return OpenMLServerException(code, message, additional) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 43513a293..59dae6deb 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -14,7 +14,7 @@ import openml import openml.utils -from ..exceptions import PyOpenMLError +from ..exceptions import PyOpenMLError, OpenMLServerNoResult from .. import config from ..flows import sklearn_to_flow, get_flow, flow_exists, _check_n_jobs, \ _copy_server_fields @@ -862,8 +862,10 @@ def list_runs(offset=None, size=None, id=None, task=None, setup=None, def _list_runs(api_call): """Helper function to parse API calls which are lists of runs""" - - xml_string = _perform_api_call(api_call) + try: + xml_string = _perform_api_call(api_call) + except OpenMLServerNoResult: + return [] runs_dict = xmltodict.parse(xml_string, force_list=('oml:run',)) # Minimalistic check if the XML is useful diff --git a/openml/runs/run.py b/openml/runs/run.py index 567533679..d52104e06 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -1,4 +1,4 @@ -from collections import OrderedDict, defaultdict +from collections import OrderedDict import json import sys import time @@ -12,6 +12,7 @@ from .._api_calls import _perform_api_call, _file_id_to_url, _read_url_files from ..exceptions import PyOpenMLError + class OpenMLRun(object): """OpenML Run: result of running a model on an openml dataset. @@ -349,6 +350,28 @@ def extract_parameters(_flow, _flow_dict, component_model, return parameters + def push_tag(self, tag): + """Annotates this run with a tag on the server. + + Parameters + ---------- + tag : str + Tag to attach to the run. + """ + data = {'run_id': self.run_id, 'tag': tag} + _perform_api_call("/run/tag", data=data) + + def remove_tag(self, tag): + """Removes a tag from this run on the server. + + Parameters + ---------- + tag : str + Tag to attach to the run. + """ + data = {'run_id': self.run_id, 'tag': tag} + _perform_api_call("/run/untag", data=data) + ################################################################################ # Functions which cannot be in runs/functions due to circular imports diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 2013f000e..e126bfc6e 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -1,3 +1,5 @@ +from time import time + from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier from sklearn.linear_model import LogisticRegression from sklearn.model_selection import RandomizedSearchCV, StratifiedKFold @@ -5,39 +7,38 @@ from openml.testing import TestBase from openml.flows.sklearn_converter import sklearn_to_flow from openml import OpenMLRun +import openml class TestRun(TestBase): - # Splitting not helpful, these test's don't rely on the server and take less - # than 1 seconds + # Splitting not helpful, these test's don't rely on the server and take + # less than 1 seconds def test_parse_parameters_flow_not_on_server(self): model = LogisticRegression() flow = sklearn_to_flow(model) - self.assertRaisesRegexp(ValueError, - 'Flow sklearn.linear_model.logistic.LogisticRegression ' - 'has no flow_id!', - OpenMLRun._parse_parameters, flow) + self.assertRaisesRegexp( + ValueError, 'Flow sklearn.linear_model.logistic.LogisticRegression' + 'has no flow_id!', OpenMLRun._parse_parameters, flow) model = AdaBoostClassifier(base_estimator=LogisticRegression()) flow = sklearn_to_flow(model) flow.flow_id = 1 - self.assertRaisesRegexp(ValueError, - 'Flow sklearn.linear_model.logistic.LogisticRegression ' - 'has no flow_id!', - OpenMLRun._parse_parameters, flow) + self.assertRaisesRegexp( + ValueError, 'Flow sklearn.linear_model.logistic.LogisticRegression' + 'has no flow_id!', OpenMLRun._parse_parameters, flow) def test_parse_parameters(self): model = RandomizedSearchCV( estimator=RandomForestClassifier(n_estimators=5), - param_distributions={"max_depth": [3, None], - "max_features": [1, 2, 3, 4], - "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10], - "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - "bootstrap": [True, False], - "criterion": ["gini", "entropy"]}, + param_distributions={ + "max_depth": [3, None], + "max_features": [1, 2, 3, 4], + "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10], + "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "bootstrap": [True, False], "criterion": ["gini", "entropy"]}, cv=StratifiedKFold(n_splits=2, random_state=1), n_iter=5) flow = sklearn_to_flow(model) @@ -49,3 +50,16 @@ def test_parse_parameters(self): if parameter['oml:name'] == 'n_estimators': self.assertEqual(parameter['oml:value'], '5') self.assertEqual(parameter['oml:component'], 2) + + def test_tagging(self): + run = openml.runs.get_run(1) + tag = "testing_tag_{}_{}".format(self.id(), time()) + run_list = openml.runs.list_runs(tag=tag) + self.assertEqual(len(run_list), 0) + run.push_tag(tag) + run_list = openml.runs.list_runs(tag=tag) + self.assertEqual(len(run_list), 1) + self.assertIn(1, run_list) + run.remove_tag(tag) + run_list = openml.runs.list_runs(tag=tag) + self.assertEqual(len(run_list), 0) From ba193edc4f1b4b70ab125976150d6392bf6b1fa1 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Thu, 12 Oct 2017 18:26:44 +0200 Subject: [PATCH 14/86] add tagging to flows --- openml/_api_calls.py | 4 ++-- openml/flows/flow.py | 24 ++++++++++++++++++++++-- openml/flows/functions.py | 22 ++++++++++++++-------- tests/test_flows/test_flow.py | 15 ++++++++++++++- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 2fee9098e..710b516c0 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -139,7 +139,7 @@ def _parse_server_exception(response): additional = None if 'oml:additional_information' in server_exception['oml:error']: additional = server_exception['oml:error']['oml:additional_information'] - if code in [370, 372, 512]: - # 512 for runs, 370 for datasets (should be 372) + if code in [370, 372, 512, 500]: + # 512 for runs, 370 for datasets (should be 372), 500 for flows return OpenMLServerNoResult(code, message, additional) return OpenMLServerException(code, message, additional) diff --git a/openml/flows/flow.py b/openml/flows/flow.py index e0f411ebe..15fd1c8ef 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -355,6 +355,28 @@ def publish(self): (flow_id, message)) return self + def push_tag(self, tag): + """Annotates this flow with a tag on the server. + + Parameters + ---------- + tag : str + Tag to attach to the flow. + """ + data = {'flow_id': self.flow_id, 'tag': tag} + _perform_api_call("/flow/tag", data=data) + + def remove_tag(self, tag): + """Removes a tag from this flow on the server. + + Parameters + ---------- + tag : str + Tag to attach to the flow. + """ + data = {'flow_id': self.flow_id, 'tag': tag} + _perform_api_call("/flow/untag", data=data) + def _copy_server_fields(source_flow, target_flow): fields_added_by_the_server = ['flow_id', 'uploader', 'version', @@ -370,5 +392,3 @@ def _copy_server_fields(source_flow, target_flow): def _add_if_nonempty(dic, key, value): if value is not None: dic[key] = value - - diff --git a/openml/flows/functions.py b/openml/flows/functions.py index b9e158a33..fd0185b4f 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -4,6 +4,7 @@ import six from openml._api_calls import _perform_api_call +from openml.exceptions import OpenMLServerNoResult from . import OpenMLFlow @@ -70,7 +71,9 @@ def list_flows(offset=None, size=None, tag=None): def flow_exists(name, external_version): - """Retrieves the flow id of the flow uniquely identified by name + external_version. + """Retrieves the flow id. + + A flow is uniquely identified by name + external_version. Parameter --------- @@ -93,8 +96,9 @@ def flow_exists(name, external_version): if not (isinstance(name, six.string_types) and len(external_version) > 0): raise ValueError('Argument \'version\' should be a non-empty string') - xml_response = _perform_api_call("flow/exists", - data={'name': name, 'external_version': external_version}) + xml_response = _perform_api_call( + "flow/exists", data={'name': name, 'external_version': + external_version}) result_dict = xmltodict.parse(xml_response) flow_id = int(result_dict['oml:flow_exists']['oml:id']) @@ -105,15 +109,17 @@ def flow_exists(name, external_version): def _list_flows(api_call): - # TODO add proper error handling here! - xml_string = _perform_api_call(api_call) + try: + xml_string = _perform_api_call(api_call) + except OpenMLServerNoResult: + return [] flows_dict = xmltodict.parse(xml_string, force_list=('oml:flow',)) # Minimalistic check if the XML is useful assert type(flows_dict['oml:flows']['oml:flow']) == list, \ type(flows_dict['oml:flows']) assert flows_dict['oml:flows']['@xmlns:oml'] == \ - 'http://openml.org/openml', flows_dict['oml:flows']['@xmlns:oml'] + 'http://openml.org/openml', flows_dict['oml:flows']['@xmlns:oml'] flows = dict() for flow_ in flows_dict['oml:flows']['oml:flow']: @@ -190,10 +196,10 @@ def assert_flows_equal(flow1, flow2, attr2 = getattr(flow2, key, None) if key == 'components': for name in set(attr1.keys()).union(attr2.keys()): - if not name in attr1: + if name not in attr1: raise ValueError('Component %s only available in ' 'argument2, but not in argument1.' % name) - if not name in attr2: + if name not in attr2: raise ValueError('Component %s only available in ' 'argument2, but not in argument1.' % name) assert_flows_equal(attr1[name], attr2[name], diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 303d890c6..04bc3936a 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -3,7 +3,7 @@ import hashlib import re import sys -import time +from time import time if sys.version_info[0] >= 3: from unittest import mock @@ -67,6 +67,19 @@ def test_get_flow(self): self.assertEqual(subflow_3.parameters['L'], '-1') self.assertEqual(len(subflow_3.components), 0) + def test_tagging(self): + flow = openml.flows.get_flow(4024) + tag = "testing_tag_{}_{}".format(self.id(), time()) + flow_list = openml.flows.list_flows(tag=tag) + self.assertEqual(len(flow_list), 0) + flow.push_tag(tag) + flow_list = openml.flows.list_flows(tag=tag) + self.assertEqual(len(flow_list), 1) + self.assertIn(4024, flow_list) + flow.remove_tag(tag) + flow_list = openml.flows.list_flows(tag=tag) + self.assertEqual(len(flow_list), 0) + def test_from_xml_to_xml(self): # Get the raw xml thing # TODO maybe get this via get_flow(), which would have to be refactored to allow getting only the xml dictionary From f39291ca6d62e6e6eecde9ee8012ea9834678570 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Thu, 12 Oct 2017 18:41:51 +0200 Subject: [PATCH 15/86] add tag pushing for tasks --- openml/_api_calls.py | 3 ++- openml/tasks/functions.py | 13 ++++++++----- openml/tasks/task.py | 24 +++++++++++++++++++++++- tests/test_tasks/test_task.py | 16 ++++++++++++++-- 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 710b516c0..b59d926bb 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -139,7 +139,8 @@ def _parse_server_exception(response): additional = None if 'oml:additional_information' in server_exception['oml:error']: additional = server_exception['oml:error']['oml:additional_information'] - if code in [370, 372, 512, 500]: + if code in [370, 372, 512, 500, 482]: # 512 for runs, 370 for datasets (should be 372), 500 for flows + # 482 for tasks return OpenMLServerNoResult(code, message, additional) return OpenMLServerException(code, message, additional) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 815339709..3bec0e7ba 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -7,7 +7,7 @@ from oslo_concurrency import lockutils import xmltodict -from ..exceptions import OpenMLCacheException +from ..exceptions import OpenMLCacheException, OpenMLServerNoResult from ..datasets import get_dataset from .task import OpenMLTask, _create_task_cache_dir from .. import config @@ -55,9 +55,9 @@ def _get_estimation_procedure_list(): Returns ------- procedures : list - A list of all estimation procedures. Every procedure is represented by a - dictionary containing the following information: id, - task type id, name, type, repeats, folds, stratified. + A list of all estimation procedures. Every procedure is represented by + a dictionary containing the following information: id, task type id, + name, type, repeats, folds, stratified. """ xml_string = _perform_api_call("estimationprocedure/list") @@ -138,7 +138,10 @@ def list_tasks(task_type_id=None, offset=None, size=None, tag=None): def _list_tasks(api_call): - xml_string = _perform_api_call(api_call) + try: + xml_string = _perform_api_call(api_call) + except OpenMLServerNoResult: + return [] tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task',)) # Minimalistic check if the XML is useful if 'oml:tasks' not in tasks_dict: diff --git a/openml/tasks/task.py b/openml/tasks/task.py index 127e7e232..d98a74424 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -4,7 +4,7 @@ from .. import config from .. import datasets from .split import OpenMLSplit -from .._api_calls import _read_url +from .._api_calls import _read_url, _perform_api_call class OpenMLTask(object): @@ -96,6 +96,28 @@ def get_split_dimensions(self): return self.split.repeats, self.split.folds, self.split.samples + def push_tag(self, tag): + """Annotates this flow with a tag on the server. + + Parameters + ---------- + tag : str + Tag to attach to the flow. + """ + data = {'flow_id': self.flow_id, 'tag': tag} + _perform_api_call("/flow/tag", data=data) + + def remove_tag(self, tag): + """Removes a tag from this flow on the server. + + Parameters + ---------- + tag : str + Tag to attach to the flow. + """ + data = {'flow_id': self.flow_id, 'tag': tag} + _perform_api_call("/flow/untag", data=data) + def _create_task_cache_dir(task_id): task_cache_dir = os.path.join(config.get_cache_directory(), "tasks", str(task_id)) diff --git a/tests/test_tasks/test_task.py b/tests/test_tasks/test_task.py index 7b95d4cec..704ce8f39 100644 --- a/tests/test_tasks/test_task.py +++ b/tests/test_tasks/test_task.py @@ -1,11 +1,11 @@ import sys -import types if sys.version_info[0] >= 3: from unittest import mock else: import mock +from time import time import numpy as np import openml @@ -45,6 +45,19 @@ def test_get_X_and_Y(self): self.assertIsInstance(Y, np.ndarray) self.assertEqual(Y.dtype, float) + def test_tagging(self): + task = openml.tasks.get_task(1) + tag = "testing_tag_{}_{}".format(self.id(), time()) + task_list = openml.tasks.list_tasks(tag=tag) + self.assertEqual(len(task_list), 0) + task.push_tag(tag) + task_list = openml.tasks.list_tasks(tag=tag) + self.assertEqual(len(task_list), 1) + self.assertIn(1, task_list) + task.remove_tag(tag) + task_list = openml.tasks.list_tasks(tag=tag) + self.assertEqual(len(task_list), 0) + def test_get_train_and_test_split_indices(self): openml.config.set_cache_directory(self.static_cache_dir) task = openml.tasks.get_task(1882) @@ -62,4 +75,3 @@ def test_get_train_and_test_split_indices(self): task.get_train_test_split_indices, 10, 0) self.assertRaisesRegexp(ValueError, "Repeat 10 not known", task.get_train_test_split_indices, 0, 10) - From c6f85b6d20accd6bca795b1014d7987340b3dcac Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 12 Oct 2017 18:56:20 +0200 Subject: [PATCH 16/86] remove argument which value can be inferred from data --- examples/OpenML_Tutorial.ipynb | 10 +++------- openml/datasets/dataset.py | 28 +++++++++++++++++++--------- openml/tasks/task.py | 12 ++---------- tests/test_datasets/test_dataset.py | 10 ++-------- 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/examples/OpenML_Tutorial.ipynb b/examples/OpenML_Tutorial.ipynb index 2632bc2ed..d670a6ead 100644 --- a/examples/OpenML_Tutorial.ipynb +++ b/examples/OpenML_Tutorial.ipynb @@ -841,7 +841,6 @@ "source": [ "X, y, attribute_names = dataset.get_data(\n", " target=dataset.default_target_attribute,\n", - " target_dtype=int,\n", " return_attribute_names=True,\n", ")\n", "eeg = pd.DataFrame(X, columns=attribute_names)\n", @@ -932,10 +931,7 @@ "from sklearn import neighbors\n", "\n", "dataset = oml.datasets.get_dataset(1471)\n", - "X, y = dataset.get_data(\n", - " target=dataset.default_target_attribute,\n", - " target_dtype=int,\n", - ")\n", + "X, y = dataset.get_data(target=dataset.default_target_attribute)\n", "clf = neighbors.KNeighborsClassifier(n_neighbors=1)\n", "clf.fit(X, y)" ] @@ -992,8 +988,8 @@ "dataset = oml.datasets.get_dataset(10)\n", "X, y, categorical = dataset.get_data(\n", " target=dataset.default_target_attribute,\n", - " target_dtype=int,\n", - " return_categorical_indicator=True)\n", + " return_categorical_indicator=True,\n", + ")\n", "print(\"Categorical features: %s\" % categorical)\n", "enc = preprocessing.OneHotEncoder(categorical_features=categorical)\n", "X = enc.fit_transform(X)\n", diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index a5b88c38b..a116f4a0e 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -184,10 +184,12 @@ def decode_arff(fh): with io.open(filename, encoding='utf8') as fh: return decode_arff(fh) - def get_data(self, target=None, target_dtype=None, include_row_id=False, + def get_data(self, target=None, + include_row_id=False, include_ignore_attributes=False, return_categorical_indicator=False, - return_attribute_names=False): + return_attribute_names=False + ): """Returns dataset content as numpy arrays / sparse matrices. Parameters @@ -225,7 +227,10 @@ def get_data(self, target=None, target_dtype=None, include_row_id=False, if not self.ignore_attributes: pass else: - to_exclude.extend(self.ignore_attributes) + if isinstance(self.ignore_attributes, six.string_types): + to_exclude.append(self.ignore_attributes) + else: + to_exclude.extend(self.ignore_attributes) if len(to_exclude) > 0: logger.info("Going to remove the following attributes:" @@ -242,14 +247,19 @@ def get_data(self, target=None, target_dtype=None, include_row_id=False, else: if isinstance(target, six.string_types): target = [target] - legal_target_types = (int, float, np.float32, np.float64) - if target_dtype not in legal_target_types: - raise ValueError( - "%s is not a legal target type. Legal target types are %s" % - (target_dtype, legal_target_types) - ) targets = np.array([True if column in target else False for column in attribute_names]) + if np.sum(targets) > 1: + raise NotImplementedError( + "Number of requested targets %d is not implemented." % + np.sum(targets) + ) + target_categorical = [ + cat for cat, column in + six.moves.zip(categorical, attribute_names) + if column in target + ] + target_dtype = int if target_categorical[0] else float try: x = data[:, ~targets] diff --git a/openml/tasks/task.py b/openml/tasks/task.py index 9617c6e94..73d0866a7 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -44,17 +44,9 @@ def get_X_and_y(self): """ dataset = self.get_dataset() - # Replace with retrieve from cache - if self.task_type_id == 1: # Supervised classification - target_dtype = int - elif self.task_type_id == 2: # Supervised regression - target_dtype = float - elif self.task_type_id == 3: # Learning curves task for classification - target_dtype = int - else: + if self.task_type_id not in (1, 2, 3): raise NotImplementedError(self.task_type) - X_and_y = dataset.get_data(target=self.target_name, - target_dtype=target_dtype) + X_and_y = dataset.get_data(target=self.target_name) return X_and_y def get_train_test_split_indices(self, fold=0, repeat=0, sample=0): diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 5de5365f0..000ffc6e8 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -47,14 +47,13 @@ def test_get_data_with_rowid(self): self.assertEqual(len(categorical), 38) def test_get_data_with_target(self): - X, y = self.dataset.get_data(target="class", target_dtype=int) + X, y = self.dataset.get_data(target="class") self.assertIsInstance(X, np.ndarray) self.assertEqual(X.dtype, np.float32) self.assertIn(y.dtype, [np.int32, np.int64]) self.assertEqual(X.shape, (898, 38)) X, y, attribute_names = self.dataset.get_data( target="class", - target_dtype=int, return_attribute_names=True ) self.assertEqual(len(attribute_names), 38) @@ -66,7 +65,6 @@ def test_get_data_rowid_and_ignore_and_target(self): self.dataset.row_id_attribute = ["hardness"] X, y = self.dataset.get_data( target="class", - target_dtype=int, include_row_id=False, include_ignore_attributes=False ) @@ -75,7 +73,6 @@ def test_get_data_rowid_and_ignore_and_target(self): self.assertEqual(X.shape, (898, 36)) X, y, categorical = self.dataset.get_data( target="class", - target_dtype=int, return_categorical_indicator=True, ) self.assertEqual(len(categorical), 36) @@ -110,7 +107,7 @@ def setUp(self): self.sparse_dataset = openml.datasets.get_dataset(4136) def test_get_sparse_dataset_with_target(self): - X, y = self.sparse_dataset.get_data(target="class", target_dtype=int) + X, y = self.sparse_dataset.get_data(target="class") self.assertTrue(sparse.issparse(X)) self.assertEqual(X.dtype, np.float32) self.assertIsInstance(y, np.ndarray) @@ -118,7 +115,6 @@ def test_get_sparse_dataset_with_target(self): self.assertEqual(X.shape, (600, 20000)) X, y, attribute_names = self.sparse_dataset.get_data( target="class", - target_dtype=int, return_attribute_names=True, ) self.assertTrue(sparse.issparse(X)) @@ -184,7 +180,6 @@ def test_get_sparse_dataset_rowid_and_ignore_and_target(self): self.sparse_dataset.row_id_attribute = ["V512"] X, y = self.sparse_dataset.get_data( target="class", - target_dtype=int, include_row_id=False, include_ignore_attributes=False, ) @@ -194,7 +189,6 @@ def test_get_sparse_dataset_rowid_and_ignore_and_target(self): self.assertEqual(X.shape, (600, 19998)) X, y, categorical = self.sparse_dataset.get_data( target="class", - target_dtype=int, return_categorical_indicator=True, ) self.assertTrue(sparse.issparse(X)) From 2bea84a422989bc0e7f2b45b48a0797b8e3c769f Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Fri, 13 Oct 2017 10:25:36 +0200 Subject: [PATCH 17/86] flow->task rename --- openml/tasks/task.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openml/tasks/task.py b/openml/tasks/task.py index d98a74424..1955637a0 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -97,26 +97,26 @@ def get_split_dimensions(self): return self.split.repeats, self.split.folds, self.split.samples def push_tag(self, tag): - """Annotates this flow with a tag on the server. + """Annotates this task with a tag on the server. Parameters ---------- tag : str - Tag to attach to the flow. + Tag to attach to the task. """ - data = {'flow_id': self.flow_id, 'tag': tag} - _perform_api_call("/flow/tag", data=data) + data = {'task_id': self.task_id, 'tag': tag} + _perform_api_call("/task/tag", data=data) def remove_tag(self, tag): - """Removes a tag from this flow on the server. + """Removes a tag from this task on the server. Parameters ---------- tag : str - Tag to attach to the flow. + Tag to attach to the task. """ - data = {'flow_id': self.flow_id, 'tag': tag} - _perform_api_call("/flow/untag", data=data) + data = {'task_id': self.task_id, 'tag': tag} + _perform_api_call("/task/untag", data=data) def _create_task_cache_dir(task_id): From b7d8b70a9809da27cd46213f558662653f6cd620 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Fri, 13 Oct 2017 10:30:28 +0200 Subject: [PATCH 18/86] don't hardcode flow number --- tests/test_flows/test_flow.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 04bc3936a..98b398a3c 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -36,8 +36,8 @@ class TestFlow(TestBase): _multiprocess_can_split_ = True def test_get_flow(self): - # We need to use the production server here because 4024 is not the test - # server + # We need to use the production server here because 4024 is not the + # test server openml.config.server = self.production_server flow = openml.flows.get_flow(4024) @@ -68,14 +68,16 @@ def test_get_flow(self): self.assertEqual(len(subflow_3.components), 0) def test_tagging(self): - flow = openml.flows.get_flow(4024) + flow_list = openml.flows.list_flows(size=1) + flow_id = list(flow_list.keys())[0] + flow = openml.flows.get_flow(flow_id) tag = "testing_tag_{}_{}".format(self.id(), time()) flow_list = openml.flows.list_flows(tag=tag) self.assertEqual(len(flow_list), 0) flow.push_tag(tag) flow_list = openml.flows.list_flows(tag=tag) self.assertEqual(len(flow_list), 1) - self.assertIn(4024, flow_list) + self.assertIn(flow_id, flow_list) flow.remove_tag(tag) flow_list = openml.flows.list_flows(tag=tag) self.assertEqual(len(flow_list), 0) From 471ca5a9c944f7ace3c3a268317016970f7cd7a8 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Fri, 13 Oct 2017 10:32:03 +0200 Subject: [PATCH 19/86] fix whitespace issue --- tests/test_runs/test_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index e126bfc6e..eccda841d 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -20,14 +20,14 @@ def test_parse_parameters_flow_not_on_server(self): flow = sklearn_to_flow(model) self.assertRaisesRegexp( ValueError, 'Flow sklearn.linear_model.logistic.LogisticRegression' - 'has no flow_id!', OpenMLRun._parse_parameters, flow) + ' has no flow_id!', OpenMLRun._parse_parameters, flow) model = AdaBoostClassifier(base_estimator=LogisticRegression()) flow = sklearn_to_flow(model) flow.flow_id = 1 self.assertRaisesRegexp( ValueError, 'Flow sklearn.linear_model.logistic.LogisticRegression' - 'has no flow_id!', OpenMLRun._parse_parameters, flow) + ' has no flow_id!', OpenMLRun._parse_parameters, flow) def test_parse_parameters(self): From 9c66b069b77ceeda5a07c68d62e87c12943d7495 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Fri, 13 Oct 2017 10:48:47 +0200 Subject: [PATCH 20/86] fix time import --- tests/test_flows/test_flow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 98b398a3c..ca0f8ce13 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -3,7 +3,7 @@ import hashlib import re import sys -from time import time +import time if sys.version_info[0] >= 3: from unittest import mock @@ -71,7 +71,7 @@ def test_tagging(self): flow_list = openml.flows.list_flows(size=1) flow_id = list(flow_list.keys())[0] flow = openml.flows.get_flow(flow_id) - tag = "testing_tag_{}_{}".format(self.id(), time()) + tag = "testing_tag_{}_{}".format(self.id(), time.time()) flow_list = openml.flows.list_flows(tag=tag) self.assertEqual(len(flow_list), 0) flow.push_tag(tag) From 7ac4f0acef32973aac503655f4d49a8d39282200 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Fri, 13 Oct 2017 11:49:05 +0200 Subject: [PATCH 21/86] update API docs --- doc/api.rst | 22 ++++++++++++++++++++++ openml/evaluations/evaluation.py | 4 ++-- openml/evaluations/functions.py | 5 +++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 79d59577c..4f4c62924 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -18,6 +18,7 @@ Top-level Classes OpenMLTask OpenMLSplit OpenMLFlow + OpenMLEvaluation :mod:`openml.datasets`: Dataset Functions @@ -43,12 +44,18 @@ Top-level Classes run_task get_run + get_runs list_runs list_runs_by_flow list_runs_by_tag list_runs_by_task list_runs_by_uploader list_runs_by_filters + run_model_on_task + run_flow_on_task + get_run_trace + initialize_model_from_run + initialize_model_from_trace :mod:`openml.tasks`: Task Functions ----------------------------------- @@ -59,6 +66,7 @@ Top-level Classes :template: function.rst get_task + get_tasks list_tasks :mod:`openml.flows`: Flow Functions @@ -68,4 +76,18 @@ Top-level Classes .. autosummary:: :toctree: generated/ :template: function.rst + + get_flow + list_flows + flow_exists + +:mod:`openml.flows`: Evaluation Functions +----------------------------------------- +.. currentmodule:: openml.evaluation + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + list_evaluations diff --git a/openml/evaluations/evaluation.py b/openml/evaluations/evaluation.py index 1a543f92c..ad7466673 100644 --- a/openml/evaluations/evaluation.py +++ b/openml/evaluations/evaluation.py @@ -21,7 +21,8 @@ class OpenMLEvaluation(object): value : float the value of this evaluation array_data : str - list of information per class (e.g., in case of precision, auroc, recall) + list of information per class (e.g., in case of precision, auroc, + recall) ''' def __init__(self, run_id, task_id, setup_id, flow_id, flow_name, data_id, data_name, function, upload_time, value, @@ -37,4 +38,3 @@ def __init__(self, run_id, task_id, setup_id, flow_id, flow_name, self.upload_time = upload_time self.value = value self.array_data = array_data - diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 5f2761c24..12b3ba9da 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -3,8 +3,9 @@ from .._api_calls import _perform_api_call from ..evaluations import OpenMLEvaluation -def list_evaluations(function, offset=None, size=None, id=None, task=None, setup=None, - flow=None, uploader=None, tag=None): + +def list_evaluations(function, offset=None, size=None, id=None, task=None, + setup=None, flow=None, uploader=None, tag=None): """List all run-evaluation pairs matching all of the given filters. Perform API call `/evaluation/function{function}/{filters} From 9e7f812f130cf6821ba9a798d7444a99561687e4 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 13 Oct 2017 13:18:57 +0200 Subject: [PATCH 22/86] Fix keyerror in quality lookup --- openml/datasets/dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 3dd78413b..f8f520624 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -455,13 +455,14 @@ def _data_features_supported(self): return True - def _check_qualities(qualities): if qualities is not None: qualities_ = {} for xmlquality in qualities: name = xmlquality['oml:name'] - if xmlquality['oml:value'] is None: + if 'oml:value' not in xmlquality: + value = float('NaN') + elif xmlquality['oml:value'] is None: value = float('NaN') elif xmlquality['oml:value'] == 'null': value = float('NaN') From 3dcc0874a61f53001ea76e878a893597807f3958 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 13 Oct 2017 13:35:28 +0200 Subject: [PATCH 23/86] include @amueller s suggestion --- openml/datasets/dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index f8f520624..5968d88c7 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -460,9 +460,10 @@ def _check_qualities(qualities): qualities_ = {} for xmlquality in qualities: name = xmlquality['oml:name'] - if 'oml:value' not in xmlquality: + + if xmlquality.get('oml:value', None) is None: value = float('NaN') - elif xmlquality['oml:value'] is None: + elif 'oml:value' not in xmlquality: value = float('NaN') elif xmlquality['oml:value'] == 'null': value = float('NaN') From 1e42e7bef27d63cdeb1e52bc6d5527a5ee5b0d19 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 13 Oct 2017 13:55:43 +0200 Subject: [PATCH 24/86] remove unnecessary line --- openml/datasets/dataset.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 5968d88c7..6440270d9 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -460,11 +460,8 @@ def _check_qualities(qualities): qualities_ = {} for xmlquality in qualities: name = xmlquality['oml:name'] - if xmlquality.get('oml:value', None) is None: value = float('NaN') - elif 'oml:value' not in xmlquality: - value = float('NaN') elif xmlquality['oml:value'] == 'null': value = float('NaN') else: From 53ff7eed13345416ffc570776db64302c1c8a385 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 12 Oct 2017 18:02:32 +0200 Subject: [PATCH 25/86] update api documentation --- doc/api.rst | 83 +++++++++++++++++++++------------ openml/evaluations/functions.py | 40 ++++++++-------- openml/flows/functions.py | 4 +- openml/runs/functions.py | 2 +- openml/runs/run.py | 8 ++-- openml/setups/functions.py | 26 +++++------ 6 files changed, 92 insertions(+), 71 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 4f4c62924..4939cd99e 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -34,6 +34,30 @@ Top-level Classes get_datasets list_datasets +:mod:`openml.evaluations`: Evaluation Functions +----------------------------------------------- +.. currentmodule:: openml.evaluations + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + list_evaluations + +:mod:`openml.flows`: Flow Functions +----------------------------------- +.. currentmodule:: openml.flows + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + flow_exists + flow_to_sklearn + get_flow + list_flows + sklearn_to_flow + :mod:`openml.runs`: Run Functions ---------------------------------- .. currentmodule:: openml.runs @@ -42,52 +66,49 @@ Top-level Classes :toctree: generated/ :template: function.rst - run_task - get_run - get_runs - list_runs - list_runs_by_flow - list_runs_by_tag - list_runs_by_task - list_runs_by_uploader - list_runs_by_filters - run_model_on_task - run_flow_on_task - get_run_trace - initialize_model_from_run - initialize_model_from_trace + get_run + get_runs + get_run_trace + initialize_model_from_run + initialize_model_from_trace + list_runs + run_model_on_task + run_flow_on_task -:mod:`openml.tasks`: Task Functions ------------------------------------ -.. currentmodule:: openml.tasks +:mod:`openml.setups`: Setup Functions +------------------------------------- +.. currentmodule:: openml.setups .. autosummary:: :toctree: generated/ :template: function.rst - get_task - get_tasks - list_tasks + get_setup + initialize_model + list_setups + setup_exists -:mod:`openml.flows`: Flow Functions ------------------------------------ -.. currentmodule:: openml.flow +:mod:`openml.study`: Study Functions +------------------------------------ +.. currentmodule:: openml.study .. autosummary:: :toctree: generated/ :template: function.rst - get_flow - list_flows - flow_exists - -:mod:`openml.flows`: Evaluation Functions ------------------------------------------ -.. currentmodule:: openml.evaluation + get_study + +:mod:`openml.tasks`: Task Functions +----------------------------------- +.. currentmodule:: openml.tasks .. autosummary:: :toctree: generated/ :template: function.rst - list_evaluations + get_task + get_tasks + list_tasks + + diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 12b3ba9da..aa7f86635 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -8,33 +8,33 @@ def list_evaluations(function, offset=None, size=None, id=None, task=None, setup=None, flow=None, uploader=None, tag=None): """List all run-evaluation pairs matching all of the given filters. - Perform API call `/evaluation/function{function}/{filters} - - Parameters - ---------- - function : str - the evaluation function. e.g., predictive_accuracy - offset : int, optional - the number of runs to skip, starting from the first - size : int, optional - the maximum number of runs to show + Perform API call ``/evaluation/function{function}/{filters}`` + + Parameters + ---------- + function : str + the evaluation function. e.g., predictive_accuracy + offset : int, optional + the number of runs to skip, starting from the first + size : int, optional + the maximum number of runs to show - id : list, optional + id : list, optional - task : list, optional + task : list, optional - setup: list, optional + setup: list, optional - flow : list, optional + flow : list, optional - uploader : list, optional + uploader : list, optional - tag : str, optional + tag : str, optional - Returns - ------- - dict - """ + Returns + ------- + dict + """ api_call = "evaluation/list/function/%s" %function if offset is not None: diff --git a/openml/flows/functions.py b/openml/flows/functions.py index fd0185b4f..bd8467a42 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -75,8 +75,8 @@ def flow_exists(name, external_version): A flow is uniquely identified by name + external_version. - Parameter - --------- + Parameters + ---------- name : string Name of the flow version : string diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 59dae6deb..8119ec72f 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -31,7 +31,7 @@ def run_model_on_task(task, model, avoid_duplicate_runs=True, flow_tags=None, seed=None): - """See ``run_flow_on_task for a documentation.""" + """See ``run_flow_on_task for a documentation``.""" flow = sklearn_to_flow(model) diff --git a/openml/runs/run.py b/openml/runs/run.py index d52104e06..4a73999d8 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -110,23 +110,23 @@ def _generate_trace_arff_dict(self): return arff_dict def get_metric_fn(self, sklearn_fn, kwargs={}): - '''Calculates metric scores based on predicted values. Assumes the + """Calculates metric scores based on predicted values. Assumes the run has been executed locally (and contains run_data). Furthermore, it assumes that the 'correct' attribute is specified in the arff (which is an optional field, but always the case for openml-python runs) Parameters - ------- + ---------- sklearn_fn : function a function pointer to a sklearn function that - accepts y_true, y_pred and *kwargs + accepts ``y_true``, ``y_pred`` and ``**kwargs`` Returns ------- scores : list a list of floats, of length num_folds * num_repeats - ''' + """ if self.data_content is not None and self.task_id is not None: predictions_arff = self._generate_arff_dict() elif 'predictions' in self.output_files: diff --git a/openml/setups/functions.py b/openml/setups/functions.py index c5c2652a7..8b667d816 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -11,8 +11,8 @@ def setup_exists(flow, model=None): ''' Checks whether a hyperparameter configuration already exists on the server. - Parameter - --------- + Parameters + ---------- flow : flow The openml flow object. @@ -77,23 +77,23 @@ def get_setup(setup_id): def list_setups(flow=None, tag=None, setup=None, offset=None, size=None): """List all setups matching all of the given filters. - Perform API call `/setup/list/{filters} + Perform API call `/setup/list/{filters}` - Parameters - ---------- - flow : int, optional + Parameters + ---------- + flow : int, optional - tag : str, optional + tag : str, optional - setup : list(int), optional + setup : list(int), optional - offset : int, optional + offset : int, optional - size : int, optional + size : int, optional - Returns - ------- - dict + Returns + ------- + dict """ api_call = "setup/list" From 8bff0b7739340446cd537871dfca3732ac1bd4fa Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 16 Oct 2017 16:21:45 +0200 Subject: [PATCH 26/86] Create LICENSE --- LICENSE | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..924b5b561 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright (c) 2014-2017, Matthias Feurer, Jan van Rijn, Andreas Müller, +Joaquin Vanschoren and others. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From c13c7b018356814241b60a5feb15c29cc294a414 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 16 Oct 2017 16:22:54 +0200 Subject: [PATCH 27/86] MAINT update license information --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f9cfeefa1..0d4377209 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ maintainer="Matthias Feurer", maintainer_email="feurerm@informatik.uni-freiburg.de", description="Python API for OpenML", - license="GPLv3", + license="BSD 3-clause", url="http://openml.org/", version=version, packages=setuptools.find_packages(), @@ -52,7 +52,7 @@ test_suite="nose.collector", classifiers=['Intended Audience :: Science/Research', 'Intended Audience :: Developers', - 'License :: GPLv3', + 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Topic :: Software Development', 'Topic :: Scientific/Engineering', From 772183568dfedfba27fa571d4399b8f5a6b5aedd Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 2 Nov 2017 14:46:43 +0100 Subject: [PATCH 28/86] ADD URL to exception --- openml/_api_calls.py | 13 +++++++++---- openml/exceptions.py | 6 +++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index b59d926bb..81a3d7756 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -95,7 +95,7 @@ def _read_url_files(url, data=None, file_dictionary=None, file_elements=None): # 'gzip,deflate' response = requests.post(url, data=data, files=file_elements) if response.status_code != 200: - raise _parse_server_exception(response) + raise _parse_server_exception(response, url=url) if 'Content-Encoding' not in response.headers or \ response.headers['Content-Encoding'] != 'gzip': warnings.warn('Received uncompressed content from OpenML for %s.' % url) @@ -117,14 +117,14 @@ def _read_url(url, data=None): response = requests.post(url, data=data) if response.status_code != 200: - raise _parse_server_exception(response) + raise _parse_server_exception(response, url=url) if 'Content-Encoding' not in response.headers or \ response.headers['Content-Encoding'] != 'gzip': warnings.warn('Received uncompressed content from OpenML for %s.' % url) return response.text -def _parse_server_exception(response): +def _parse_server_exception(response, url=None): # OpenML has a sopisticated error system # where information about failures is provided. try to parse this try: @@ -143,4 +143,9 @@ def _parse_server_exception(response): # 512 for runs, 370 for datasets (should be 372), 500 for flows # 482 for tasks return OpenMLServerNoResult(code, message, additional) - return OpenMLServerException(code, message, additional) + return OpenMLServerException( + code=code, + message=message, + additional=additional, + url=url + ) diff --git a/openml/exceptions.py b/openml/exceptions.py index eb5890a1c..386e25cdc 100644 --- a/openml/exceptions.py +++ b/openml/exceptions.py @@ -16,9 +16,13 @@ class OpenMLServerException(OpenMLServerError): """exception for when the result of the server was not 200 (e.g., listing call w/o results). """ - def __init__(self, code, message, additional=None): + # Code needs to be optional to allow the exceptino to be picklable: + # https://stackoverflow.com/questions/16244923/how-to-make-a-custom-exception-class-with-multiple-init-args-pickleable + def __init__(self, message, code=None, additional=None, url=None): + self.message = message self.code = code self.additional = additional + self.url = url super(OpenMLServerException, self).__init__(message) From 34d5a96b61eb4e143db3142f8ebda787c2902fd0 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 2 Nov 2017 14:47:23 +0100 Subject: [PATCH 29/86] test md5 hash on dataset download --- openml/datasets/functions.py | 11 +++++++++++ tests/test_datasets/test_dataset_functions.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 9f7fa4e71..f6dea2cfb 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -1,4 +1,5 @@ from collections import OrderedDict +import hashlib import io import os import re @@ -365,6 +366,8 @@ def _get_dataset_arff(did_cache_dir, description): Location of arff file. """ output_file_path = os.path.join(did_cache_dir, "dataset.arff") + md5_checksum_fixture = description.get("oml:md5_checksum") + did = description.get("oml:id") # This means the file is still there; whether it is useful is up to # the user and not checked by the program. @@ -377,6 +380,14 @@ def _get_dataset_arff(did_cache_dir, description): url = description['oml:url'] arff_string = _read_url(url) + md5 = hashlib.md5() + md5.update(arff_string.encode('utf8')) + md5_checksum = md5.hexdigest() + if md5_checksum != md5_checksum_fixture: + raise ValueError( + 'Checksum %s of downloaded dataset %d is unequal to the checksum ' + '%s sent by the server.' % (md5_checksum, did, md5_checksum_fixture) + ) with io.open(output_file_path, "w", encoding='utf8') as fh: fh.write(arff_string) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 337efc55b..5a0520a46 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -200,6 +200,20 @@ def test__getarff_path_dataset_arff(self): self.assertIsInstance(arff_path, str) self.assertTrue(os.path.exists(arff_path)) + def test__getarff_md5_issue(self): + description = { + 'oml:id': 5, + 'oml:md5_checksum': 'abc', + 'oml:url': 'https://www.openml.org/data/download/61', + } + self.assertRaisesRegexp( + ValueError, + 'Checksum ad484452702105cbf3d30f8deaba39a9 of downloaded dataset 5 ' + 'is unequal to the checksum abc sent by the server.', + _get_dataset_arff, + self.workdir, description, + ) + def test__get_dataset_features(self): features = _get_dataset_features(self.workdir, 2) self.assertIsInstance(features, dict) From 88683bc0c9dce9aed744e4f74d7b2ec7b07f9666 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Thu, 9 Nov 2017 17:55:30 -0500 Subject: [PATCH 30/86] ensure features are represented nicely in jupyter / ipython, which abuses __repr__ --- openml/datasets/data_feature.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/openml/datasets/data_feature.py b/openml/datasets/data_feature.py index 0254d4624..627d92745 100644 --- a/openml/datasets/data_feature.py +++ b/openml/datasets/data_feature.py @@ -16,11 +16,13 @@ class OpenMLDataFeature(object): """ LEGAL_DATA_TYPES = ['nominal', 'numeric', 'string', 'date'] - def __init__(self, index, name, data_type, nominal_values, number_missing_values): + def __init__(self, index, name, data_type, nominal_values, + number_missing_values): if type(index) != int: raise ValueError('Index is of wrong datatype') if data_type not in self.LEGAL_DATA_TYPES: - raise ValueError('data type should be in %s, found: %s' %(str(self.LEGAL_DATA_TYPES),data_type)) + raise ValueError('data type should be in %s, found: %s' % + (str(self.LEGAL_DATA_TYPES), data_type)) if nominal_values is not None and type(nominal_values) != list: raise ValueError('Nominal_values is of wrong datatype') if type(number_missing_values) != int: @@ -33,4 +35,7 @@ def __init__(self, index, name, data_type, nominal_values, number_missing_values self.number_missing_values = number_missing_values def __str__(self): - return "[%d - %s (%s)]" %(self.index, self.name, self.data_type) \ No newline at end of file + return "[%d - %s (%s)]" % (self.index, self.name, self.data_type) + + def _repr_pretty_(self, pp, cycle): + pp.text(str(self)) From 391ab1e0ef1e7913c178f8b6fc8498d716147f93 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Thu, 16 Nov 2017 17:51:34 +0100 Subject: [PATCH 31/86] requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e5aa16739..f928fe964 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ xmltodict nose requests scikit-learn>=0.18 +nbconvert nbformat python-dateutil -oslo.concurrency \ No newline at end of file +oslo.concurrency From bc982bebfbff1ba5251f27e99b68eeaf5cc6516b Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Fri, 17 Nov 2017 11:06:20 +0100 Subject: [PATCH 32/86] changed check to six string type #375 --- openml/datasets/dataset.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 6440270d9..4c98d2e64 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -373,13 +373,17 @@ def get_features_by_type(self, data_type, exclude=None, result : list a list of indices that have the specified data type ''' - assert data_type in OpenMLDataFeature.LEGAL_DATA_TYPES, "Illegal feature type requested" + if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES: + raise TypeError("Illegal feature type requested") if self.ignore_attributes is not None: - assert type(self.ignore_attributes) is list, "ignore_attributes should be a list" + if not isinstance(self.ignore_attributes, list): + raise TypeError("ignore_attributes should be a list") if self.row_id_attribute is not None: - assert type(self.row_id_attribute) is str, "row id attribute should be a str" + if not isinstance(self.row_id_attribute, six.string_types): + raise TypeError("row id attribute should be a str") if exclude is not None: - assert type(exclude) is list, "Exclude should be a list" + if not isinstance(exclude, list): + raise TypeError("Exclude should be a list") # assert all(isinstance(elem, str) for elem in exclude), "Exclude should be a list of strings" to_exclude = [] if exclude is not None: From 7ec40452fb71ce9c08de36902f607391415a16ce Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Nov 2017 19:26:45 +0100 Subject: [PATCH 33/86] Modularized a complex run function, for custom experimentation. Now the internal models could potentially be accessed by the outside world. --- openml/runs/functions.py | 234 +++++++++++++++++++++++++-------------- 1 file changed, 149 insertions(+), 85 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 8119ec72f..29a1f1b72 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -89,8 +89,7 @@ def run_flow_on_task(task, flow, avoid_duplicate_runs=True, flow_tags=None, dataset = task.get_dataset() - class_labels = task.class_labels - if class_labels is None: + if task.class_labels is None: raise ValueError('The task has no class labels. This method currently ' 'only works for tasks with class labels.') @@ -98,7 +97,7 @@ def run_flow_on_task(task, flow, avoid_duplicate_runs=True, flow_tags=None, tags = ['openml-python', run_environment[1]] # execute the run - res = _run_task_get_arffcontent(flow.model, task, class_labels) + res = _run_task_get_arffcontent(flow.model, task) if flow.flow_id is None: _publish_flow_if_necessary(flow) @@ -359,8 +358,7 @@ def _prediction_to_row(rep_no, fold_no, sample_no, row_id, correct_label, return arff_line -# JvR: why is class labels a parameter? could be removed and taken from task object, right? -def _run_task_get_arffcontent(model, task, class_labels): +def _run_task_get_arffcontent(model, task): def _prediction_to_probabilities(y, model_classes): # y: list or numpy array of predictions @@ -373,18 +371,17 @@ def _prediction_to_probabilities(y, model_classes): result[obs][array_idx] = 1.0 return result - X, Y = task.get_X_and_y() arff_datacontent = [] arff_tracecontent = [] # stores fold-based evaluation measures. In case of a sample based task, # this information is multiple times overwritten, but due to the ordering # of tne loops, eventually it contains the information based on the full # dataset size - user_defined_measures_fold = defaultdict(lambda: defaultdict(dict)) + user_defined_measures_per_fold = defaultdict(lambda: defaultdict(dict)) # stores sample-based evaluation measures (sublevel of fold-based) # will also be filled on a non sample-based task, but the information # is the same as the fold-based measures, and disregarded in that case - user_defined_measures_sample = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) + user_defined_measures_per_sample = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) # sys.version_info returns a tuple, the following line compares the entry of tuples # https://docs.python.org/3.6/reference/expressions.html#value-comparisons @@ -397,83 +394,20 @@ def _prediction_to_probabilities(y, model_classes): for fold_no in range(num_folds): for sample_no in range(num_samples): model_fold = sklearn.base.clone(model, safe=True) - train_indices, test_indices = task.get_train_test_split_indices(repeat=rep_no, - fold=fold_no, - sample=sample_no) - trainX = X[train_indices] - trainY = Y[train_indices] - testX = X[test_indices] - testY = Y[test_indices] - - try: - # for measuring runtime. Only available since Python 3.3 - if can_measure_runtime: - modelfit_starttime = time.process_time() - model_fold.fit(trainX, trainY) - - if can_measure_runtime: - modelfit_duration = (time.process_time() - modelfit_starttime) * 1000 - user_defined_measures_sample['usercpu_time_millis_training'][rep_no][fold_no][sample_no] = modelfit_duration - user_defined_measures_fold['usercpu_time_millis_training'][rep_no][fold_no] = modelfit_duration - except AttributeError as e: - # typically happens when training a regressor on classification task - raise PyOpenMLError(str(e)) - - # extract trace, if applicable - if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): - arff_tracecontent.extend(_extract_arfftrace(model_fold, rep_no, fold_no)) - - # search for model classes_ (might differ depending on modeltype) - # first, pipelines are a special case (these don't have a classes_ - # object, but rather borrows it from the last step. We do this manually, - # because of the BaseSearch check) - if isinstance(model_fold, sklearn.pipeline.Pipeline): - used_estimator = model_fold.steps[-1][-1] - else: - used_estimator = model_fold + res =_run_model_on_fold(model_fold, task, rep_no, fold_no, sample_no, can_measure_runtime) + arff_datacontent_fold, arff_tracecontent_fold, user_defined_measures_fold, model_fold = res - if isinstance(used_estimator, sklearn.model_selection._search.BaseSearchCV): - model_classes = used_estimator.best_estimator_.classes_ - else: - model_classes = used_estimator.classes_ - - if can_measure_runtime: - modelpredict_starttime = time.process_time() - - PredY = model_fold.predict(testX) - try: - ProbaY = model_fold.predict_proba(testX) - except AttributeError: - ProbaY = _prediction_to_probabilities(PredY, list(model_classes)) - - # add client-side calculated metrics. These might be used on the server as consistency check - def _calculate_local_measure(sklearn_fn, openml_name): - user_defined_measures_fold[openml_name][rep_no][fold_no] = \ - sklearn_fn(testY, PredY) - user_defined_measures_sample[openml_name][rep_no][fold_no][sample_no] = \ - sklearn_fn(testY, PredY) - - _calculate_local_measure(sklearn.metrics.accuracy_score, 'predictive_accuracy') - - if can_measure_runtime: - modelpredict_duration = (time.process_time() - modelpredict_starttime) * 1000 - user_defined_measures_fold['usercpu_time_millis_testing'][rep_no][fold_no] = modelpredict_duration - user_defined_measures_fold['usercpu_time_millis'][rep_no][fold_no] = modelfit_duration + modelpredict_duration - user_defined_measures_sample['usercpu_time_millis_testing'][rep_no][fold_no][sample_no] = modelpredict_duration - user_defined_measures_sample['usercpu_time_millis'][rep_no][fold_no][sample_no] = modelfit_duration + modelpredict_duration - - if ProbaY.shape[1] != len(class_labels): - warnings.warn("Repeat %d Fold %d: estimator only predicted for %d/%d classes!" %(rep_no, fold_no, ProbaY.shape[1], len(class_labels))) - - for i in range(0, len(test_indices)): - arff_line = _prediction_to_row(rep_no, fold_no, sample_no, - test_indices[i], class_labels[testY[i]], - PredY[i], ProbaY[i], class_labels, model_classes) - arff_datacontent.append(arff_line) - - if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): + arff_datacontent.extend(arff_datacontent_fold) + arff_tracecontent.extend(arff_tracecontent_fold) + + for measure in user_defined_measures_fold: + user_defined_measures_per_fold[measure][rep_no][fold_no] = user_defined_measures_fold[measure] + user_defined_measures_per_sample[measure][rep_no][fold_no][sample_no] = user_defined_measures_fold[measure] + + + if isinstance(model, sklearn.model_selection._search.BaseSearchCV): # arff_tracecontent is already set - arff_trace_attributes = _extract_arfftrace_attributes(model_fold) + arff_trace_attributes = _extract_arfftrace_attributes(model) else: arff_tracecontent = None arff_trace_attributes = None @@ -481,8 +415,138 @@ def _calculate_local_measure(sklearn_fn, openml_name): return arff_datacontent, \ arff_tracecontent, \ arff_trace_attributes, \ - user_defined_measures_fold, \ - user_defined_measures_sample + user_defined_measures_per_fold, \ + user_defined_measures_per_sample + + +def _run_model_on_fold(model, task, rep_no, fold_no, sample_no, can_measure_runtime): + """Internal function that executes a model on a fold (and possibly + subsample) of the dataset. It returns the data that is necessary + to construct the OpenML Run object (potentially over more than + one folds). Is used by run_task_get_arff_content. Do not use this + function unless you know what you are doing. + + Parameters + ---------- + model : sklearn model + The UNTRAINED model to run + task : OpenMLTask + The task to run the model on + rep_no : int + The repeat of the experiment (0-based; in case of 1 time CV, + always 0) + fold_no : int + The fold nr of the experiment (0-based; in case of holdout, + always 0) + sample_no : int + In case of learning curves, the index of the subsample (0-based; + in case of no learning curve, always 0) + can_measure_runtime : bool + Wether we are allowed to measure runtime (requires: Single node + computation and Python >= 3.3) + + Returns + ------- + arff_datacontent : List[List] + Arff representation (list of lists) of the predictions that were + generated by this fold (for putting in predictions.arff) + arff_tracecontent : List[List] + Arff representation (list of lists) of the trace data that was + generated by this fold (for putting in trace.arff) + user_defined_measures : Dict[float] + User defined measures that were generated on this fold + model : sklearn model + The model trained on this fold + """ + def _prediction_to_probabilities(y, model_classes): + # y: list or numpy array of predictions + # model_classes: sklearn classifier mapping from original array id to prediction index id + if not isinstance(model_classes, list): + raise ValueError('please convert model classes to list prior to calling this fn') + result = np.zeros((len(y), len(model_classes)), dtype=np.float32) + for obs, prediction_idx in enumerate(y): + array_idx = model_classes.index(prediction_idx) + result[obs][array_idx] = 1.0 + return result + + # TODO: if possible, give a warning if model is already fitted (acceptable in case of custom experimentation, + # but not desirable if we want to upload to OpenML). + + train_indices, test_indices = task.get_train_test_split_indices(repeat=rep_no, + fold=fold_no, + sample=sample_no) + + X, Y = task.get_X_and_y() + trainX = X[train_indices] + trainY = Y[train_indices] + testX = X[test_indices] + testY = Y[test_indices] + user_defined_measures = dict() + + try: + # for measuring runtime. Only available since Python 3.3 + if can_measure_runtime: + modelfit_starttime = time.process_time() + model.fit(trainX, trainY) + + if can_measure_runtime: + modelfit_duration = (time.process_time() - modelfit_starttime) * 1000 + user_defined_measures['usercpu_time_millis_training'] = modelfit_duration + except AttributeError as e: + # typically happens when training a regressor on classification task + raise PyOpenMLError(str(e)) + + # extract trace, if applicable + arff_tracecontent = [] + if isinstance(model, sklearn.model_selection._search.BaseSearchCV): + arff_tracecontent.extend(_extract_arfftrace(model, rep_no, fold_no)) + + # search for model classes_ (might differ depending on modeltype) + # first, pipelines are a special case (these don't have a classes_ + # object, but rather borrows it from the last step. We do this manually, + # because of the BaseSearch check) + if isinstance(model, sklearn.pipeline.Pipeline): + used_estimator = model.steps[-1][-1] + else: + used_estimator = model + + if isinstance(used_estimator, sklearn.model_selection._search.BaseSearchCV): + model_classes = used_estimator.best_estimator_.classes_ + else: + model_classes = used_estimator.classes_ + + if can_measure_runtime: + modelpredict_starttime = time.process_time() + + PredY = model.predict(testX) + try: + ProbaY = model.predict_proba(testX) + except AttributeError: + ProbaY = _prediction_to_probabilities(PredY, list(model_classes)) + + # add client-side calculated metrics. These might be used on the server as consistency check + def _calculate_local_measure(sklearn_fn, openml_name): + user_defined_measures[openml_name] = sklearn_fn(testY, PredY) + + _calculate_local_measure(sklearn.metrics.accuracy_score, 'predictive_accuracy') + + if can_measure_runtime: + modelpredict_duration = (time.process_time() - modelpredict_starttime) * 1000 + user_defined_measures['usercpu_time_millis_testing'] = modelpredict_duration + user_defined_measures['usercpu_time_millis'] = modelfit_duration + modelpredict_duration + + + if ProbaY.shape[1] != len(task.class_labels): + warnings.warn("Repeat %d Fold %d: estimator only predicted for %d/%d classes!" % ( + rep_no, fold_no, ProbaY.shape[1], len(task.class_labels))) + + arff_datacontent = [] + for i in range(0, len(test_indices)): + arff_line = _prediction_to_row(rep_no, fold_no, sample_no, + test_indices[i], task.class_labels[testY[i]], + PredY[i], ProbaY[i], task.class_labels, model_classes) + arff_datacontent.append(arff_line) + return arff_datacontent, arff_tracecontent, user_defined_measures, model def _extract_arfftrace(model, rep_no, fold_no): From 42f2f1dab77fd103809c8b14952d94117f7f68c8 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Nov 2017 19:26:59 +0100 Subject: [PATCH 34/86] unit test fix --- tests/test_runs/test_run_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 7606d3ac6..94993c1ae 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -720,7 +720,7 @@ def test__run_task_get_arffcontent(self): num_repeats = 1 clf = SGDClassifier(loss='log', random_state=1) - res = openml.runs.functions._run_task_get_arffcontent(clf, task, class_labels) + res = openml.runs.functions._run_task_get_arffcontent(clf, task) arff_datacontent, arff_tracecontent, _, fold_evaluations, sample_evaluations = res # predictions self.assertIsInstance(arff_datacontent, list) From 4901cfee7ebb2d4cd67db0644644efffd253d1cb Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Nov 2017 19:35:05 +0100 Subject: [PATCH 35/86] small fixes --- openml/runs/functions.py | 7 ++++--- tests/test_runs/test_run_functions.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 29a1f1b72..d6df31fa4 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -404,10 +404,11 @@ def _prediction_to_probabilities(y, model_classes): user_defined_measures_per_fold[measure][rep_no][fold_no] = user_defined_measures_fold[measure] user_defined_measures_per_sample[measure][rep_no][fold_no][sample_no] = user_defined_measures_fold[measure] - - if isinstance(model, sklearn.model_selection._search.BaseSearchCV): + # Note that we need to use a fitted model (i.e., model_fold, and not model) here, + # to ensure it contains the hyperparameter data (in cv_results_) + if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): # arff_tracecontent is already set - arff_trace_attributes = _extract_arfftrace_attributes(model) + arff_trace_attributes = _extract_arfftrace_attributes(model_fold) else: arff_tracecontent = None arff_trace_attributes = None diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 94993c1ae..aef62ebdd 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -917,8 +917,8 @@ def test_predict_proba_hardclassifier(self): ('imputer', sklearn.preprocessing.Imputer()), ('estimator', HardNaiveBayes()) ]) - arff_content1, arff_header1, _, _, _ = _run_task_get_arffcontent(clf1, task, task.class_labels) - arff_content2, arff_header2, _, _, _ = _run_task_get_arffcontent(clf2, task, task.class_labels) + arff_content1, arff_header1, _, _, _ = _run_task_get_arffcontent(clf1, task) + arff_content2, arff_header2, _, _, _ = _run_task_get_arffcontent(clf2, task) # verifies last two arff indices (predict and correct) # TODO: programmatically check wether these are indeed features (predict, correct) From 3bd961caf8e201c88595c4e758f64add0209c2cc Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Nov 2017 19:40:25 +0100 Subject: [PATCH 36/86] unit test fix --- tests/test_runs/test_run_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index aef62ebdd..9c5e1d1b3 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -896,7 +896,7 @@ def test_run_on_dataset_with_missing_labels(self): model = Pipeline(steps=[('Imputer', Imputer(strategy='median')), ('Estimator', DecisionTreeClassifier())]) - data_content, _, _, _, _ = _run_task_get_arffcontent(model, task, class_labels) + data_content, _, _, _, _ = _run_task_get_arffcontent(model, task) # 2 folds, 5 repeats; keep in mind that this task comes from the test # server, the task on the live server is different self.assertEqual(len(data_content), 4490) From 017af9b3008d028d36a2804a25eec937286306e8 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Wed, 22 Nov 2017 09:36:39 +0100 Subject: [PATCH 37/86] unrelated bugfix --- openml/runs/functions.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index d6df31fa4..838c79120 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -99,7 +99,7 @@ def run_flow_on_task(task, flow, avoid_duplicate_runs=True, flow_tags=None, # execute the run res = _run_task_get_arffcontent(flow.model, task) - if flow.flow_id is None: + if not isinstance(flow.flow_id, int): _publish_flow_if_necessary(flow) run = OpenMLRun(task_id=task.task_id, flow_id=flow.flow_id, @@ -222,13 +222,17 @@ def initialize_model_from_trace(run_id, repeat, fold, iteration=None): def _run_exists(task_id, setup_id): - ''' - Checks whether a task/setup combination is already present on the server. + """Checks whether a task/setup combination is already present on the server. - :param task_id: int - :param setup_id: int - :return: List of run ids iff these already exists on the server, False otherwise - ''' + Parameters + ---------- + task_id: int + setup_id: int + + Returns + ------- + List of run ids iff these already exists on the server, False otherwise + """ if setup_id <= 0: # openml setups are in range 1-inf return False @@ -246,7 +250,7 @@ def _run_exists(task_id, setup_id): def _get_seeded_model(model, seed=None): - '''Sets all the non-seeded components of a model with a seed. + """Sets all the non-seeded components of a model with a seed. Models that are already seeded will maintain the seed. In this case, only integer seeds are allowed (An exception is thrown when a RandomState was used as seed) @@ -264,7 +268,7 @@ def _get_seeded_model(model, seed=None): model : sklearn model a version of the model where all (sub)components have a seed - ''' + """ def _seed_current_object(current_value): if isinstance(current_value, int): # acceptable behaviour @@ -525,21 +529,19 @@ def _prediction_to_probabilities(y, model_classes): except AttributeError: ProbaY = _prediction_to_probabilities(PredY, list(model_classes)) - # add client-side calculated metrics. These might be used on the server as consistency check - def _calculate_local_measure(sklearn_fn, openml_name): - user_defined_measures[openml_name] = sklearn_fn(testY, PredY) - - _calculate_local_measure(sklearn.metrics.accuracy_score, 'predictive_accuracy') - if can_measure_runtime: modelpredict_duration = (time.process_time() - modelpredict_starttime) * 1000 user_defined_measures['usercpu_time_millis_testing'] = modelpredict_duration user_defined_measures['usercpu_time_millis'] = modelfit_duration + modelpredict_duration - if ProbaY.shape[1] != len(task.class_labels): - warnings.warn("Repeat %d Fold %d: estimator only predicted for %d/%d classes!" % ( - rep_no, fold_no, ProbaY.shape[1], len(task.class_labels))) + warnings.warn("Repeat %d Fold %d: estimator only predicted for %d/%d classes!" % (rep_no, fold_no, ProbaY.shape[1], len(task.class_labels))) + + # add client-side calculated metrics. These might be used on the server as consistency check + def _calculate_local_measure(sklearn_fn, openml_name): + user_defined_measures[openml_name] = sklearn_fn(testY, PredY) + + _calculate_local_measure(sklearn.metrics.accuracy_score, 'predictive_accuracy') arff_datacontent = [] for i in range(0, len(test_indices)): From 728877ec097525d6d69319d64880d1f75e40fd07 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Wed, 22 Nov 2017 09:39:23 +0100 Subject: [PATCH 38/86] flow --- openml/runs/functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 838c79120..9c0fe0a9f 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -99,7 +99,8 @@ def run_flow_on_task(task, flow, avoid_duplicate_runs=True, flow_tags=None, # execute the run res = _run_task_get_arffcontent(flow.model, task) - if not isinstance(flow.flow_id, int): + # in case the flow not exists, we will get a "False" back (which can be seen as an int instance) + if not isinstance(flow.flow_id, int) or flow_id == False: _publish_flow_if_necessary(flow) run = OpenMLRun(task_id=task.task_id, flow_id=flow.flow_id, From d785e241b4b9d5782df6a71ec7307c9a669a2322 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Wed, 22 Nov 2017 09:39:48 +0100 Subject: [PATCH 39/86] same --- openml/runs/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 9c0fe0a9f..4edccb35d 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -100,7 +100,7 @@ def run_flow_on_task(task, flow, avoid_duplicate_runs=True, flow_tags=None, res = _run_task_get_arffcontent(flow.model, task) # in case the flow not exists, we will get a "False" back (which can be seen as an int instance) - if not isinstance(flow.flow_id, int) or flow_id == False: + if not isinstance(flow.flow_id, int) or flow_id is False: _publish_flow_if_necessary(flow) run = OpenMLRun(task_id=task.task_id, flow_id=flow.flow_id, From 49637acb0eeb8a44dc7b9dfba765ced8dc8977d7 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Fri, 24 Nov 2017 18:23:54 +0100 Subject: [PATCH 40/86] also made create_run_from_xml more modular --- openml/runs/functions.py | 67 ++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 4edccb35d..ed7e8908e 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -99,8 +99,8 @@ def run_flow_on_task(task, flow, avoid_duplicate_runs=True, flow_tags=None, # execute the run res = _run_task_get_arffcontent(flow.model, task) - # in case the flow not exists, we will get a "False" back (which can be seen as an int instance) - if not isinstance(flow.flow_id, int) or flow_id is False: + # in case the flow not exists, we will get a "False" back (which can be + if not isinstance(flow.flow_id, int) or flow_id == False: _publish_flow_if_necessary(flow) run = OpenMLRun(task_id=task.task_id, flow_id=flow.flow_id, @@ -654,7 +654,7 @@ def get_run(run_id): return run -def _create_run_from_xml(xml): +def _create_run_from_xml(xml, from_server=True): """Create a run object from xml returned from server. Parameters @@ -667,21 +667,35 @@ def _create_run_from_xml(xml): run : OpenMLRun New run object representing run_xml. """ + + def obtain_field(xml_obj, fieldname, from_server, cast=None): + if fieldname in xml_obj: + if cast is not None: + return cast(xml_obj[fieldname]) + return xml_obj[fieldname] + elif not from_server: + return None + else: + raise AttributeError('Run XML does not contain required (server) field: ', fieldname) + run = xmltodict.parse(xml)["oml:run"] - run_id = int(run['oml:run_id']) - uploader = int(run['oml:uploader']) - uploader_name = run['oml:uploader_name'] + run_id = obtain_field(run, 'oml:run_id', from_server, cast=int) + uploader = obtain_field(run, 'oml:uploader', from_server, cast=int) + uploader_name = obtain_field(run, 'oml:uploader_name', from_server) task_id = int(run['oml:task_id']) - task_type = run['oml:task_type'] + task_type = obtain_field(run, 'oml:task_type', from_server) + + # even with the server requirement this field may be empty. if 'oml:task_evaluation_measure' in run: task_evaluation_measure = run['oml:task_evaluation_measure'] else: task_evaluation_measure = None + flow_id = int(run['oml:flow_id']) - flow_name = run['oml:flow_name'] - setup_id = int(run['oml:setup_id']) - setup_string = run['oml:setup_string'] + flow_name = obtain_field(run, 'oml:flow_name', from_server) + setup_id = obtain_field(run, 'oml:setup_id', from_server, cast=int) + setup_string = obtain_field(run, 'oml:setup_string', from_server) parameters = dict() if 'oml:parameter_settings' in run: @@ -691,7 +705,10 @@ def _create_run_from_xml(xml): value = parameter_dict['oml:value'] parameters[key] = value - dataset_id = int(run['oml:input_data']['oml:dataset']['oml:did']) + if 'oml:input_data' in run: + dataset_id = int(run['oml:input_data']['oml:dataset']['oml:did']) + elif not from_server: + dataset_id = None files = dict() evaluations = dict() @@ -700,21 +717,23 @@ def _create_run_from_xml(xml): if 'oml:output_data' not in run: raise ValueError('Run does not contain output_data (OpenML server error?)') else: - if isinstance(run['oml:output_data']['oml:file'], dict): - # only one result.. probably due to an upload error - file_dict = run['oml:output_data']['oml:file'] - files[file_dict['oml:name']] = int(file_dict['oml:file_id']) - elif isinstance(run['oml:output_data']['oml:file'], list): - # multiple files, the normal case - for file_dict in run['oml:output_data']['oml:file']: + output_data = run['oml:output_data'] + if 'oml:file' in output_data: + if isinstance(output_data['oml:file'], dict): + # only one result.. probably due to an upload error + file_dict = output_data['oml:file'] files[file_dict['oml:name']] = int(file_dict['oml:file_id']) - else: - raise TypeError(type(run['oml:output_data']['oml:file'])) + elif isinstance(output_data['oml:file'], list): + # multiple files, the normal case + for file_dict in output_data['oml:file']: + files[file_dict['oml:name']] = int(file_dict['oml:file_id']) + else: + raise TypeError(type(output_data['oml:file'])) - if 'oml:evaluation' in run['oml:output_data']: + if 'oml:evaluation' in output_data: # in normal cases there should be evaluations, but in case there # was an error these could be absent - for evaluation_dict in run['oml:output_data']['oml:evaluation']: + for evaluation_dict in output_data['oml:evaluation']: key = evaluation_dict['oml:name'] if 'oml:value' in evaluation_dict: value = float(evaluation_dict['oml:value']) @@ -740,11 +759,11 @@ def _create_run_from_xml(xml): else: evaluations[key] = value - if 'description' not in files: + if 'description' not in files and from_server is True: raise ValueError('No description file for run %d in run ' 'description XML' % run_id) - if 'predictions' not in files: + if 'predictions' not in files and from_server is True: task = openml.tasks.get_task(task_id) if task.task_type_id == 8: raise NotImplementedError( From 09126241ab3363f8ca80b1adcfd51eb32ba3e1d9 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Mon, 27 Nov 2017 19:34:45 +0100 Subject: [PATCH 41/86] added comment to the internal function --- openml/runs/functions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index ed7e8908e..3196a5bdb 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -669,6 +669,9 @@ def _create_run_from_xml(xml, from_server=True): """ def obtain_field(xml_obj, fieldname, from_server, cast=None): + # this function can be used to check whether a field is present in an object. + # if it is not present, either returns None or throws an error (this is + # usually done if the xml comes from the server) if fieldname in xml_obj: if cast is not None: return cast(xml_obj[fieldname]) From c80c1b0d856efd43b08a954eeb0661dcf1e35f85 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Mon, 27 Nov 2017 21:51:41 +0100 Subject: [PATCH 42/86] made sure we can assume tasp inputs is a list #383 --- openml/tasks/functions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 3bec0e7ba..b446ad8d7 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -176,7 +176,11 @@ def _list_tasks(api_call): 'status': task_['oml:status']} # Other task inputs - for input in task_.get('oml:input', list()): + task_inputs = task_.get('oml:input') + if isinstance(task_inputs, dict): + task_inputs = [task_inputs] + + for input in task_inputs: if input['@name'] == 'estimation_procedure': task[input['@name']] = proc_dict[int(input['#text'])]['name'] else: From a423a6ab60b9a8b3691887df4eba2ad0d973f0ee Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Mon, 27 Nov 2017 21:57:24 +0100 Subject: [PATCH 43/86] added unit test --- tests/test_tasks/test_task_functions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 0eba2b8a7..4d8b7ce2a 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -42,6 +42,12 @@ def test__get_estimation_procedure_list(self): self.assertIsInstance(estimation_procedures[0], dict) self.assertEqual(estimation_procedures[0]['task_type_id'], 1) + + def test_list_clustering_task(self): + # as shown by #383, clustering tasks can give problems to server + openml.config.server = self.production_server + openml.tasks.list_tasks(task_type_id=5, size=10) + def _check_task(self, task): self.assertEqual(type(task), dict) self.assertGreaterEqual(len(task), 2) From 2bda55bccad528b0b756b428a3f7d2a182b7bf59 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 28 Nov 2017 14:36:54 +0100 Subject: [PATCH 44/86] explained unit test, used force_list option to cast to list --- openml/tasks/functions.py | 8 ++------ tests/test_tasks/test_task_functions.py | 5 ++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index b446ad8d7..31a76eb48 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -142,7 +142,7 @@ def _list_tasks(api_call): xml_string = _perform_api_call(api_call) except OpenMLServerNoResult: return [] - tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task',)) + tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task','oml:input')) # Minimalistic check if the XML is useful if 'oml:tasks' not in tasks_dict: raise ValueError('Error in return XML, does not contain "oml:runs": %s' @@ -176,11 +176,7 @@ def _list_tasks(api_call): 'status': task_['oml:status']} # Other task inputs - task_inputs = task_.get('oml:input') - if isinstance(task_inputs, dict): - task_inputs = [task_inputs] - - for input in task_inputs: + for input in task_.get('oml:input', list()): if input['@name'] == 'estimation_procedure': task[input['@name']] = proc_dict[int(input['#text'])]['name'] else: diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 4d8b7ce2a..ea84a27c7 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -42,11 +42,11 @@ def test__get_estimation_procedure_list(self): self.assertIsInstance(estimation_procedures[0], dict) self.assertEqual(estimation_procedures[0]['task_type_id'], 1) - def test_list_clustering_task(self): - # as shown by #383, clustering tasks can give problems to server + # as shown by #383, clustering tasks can give list/dict casting problems openml.config.server = self.production_server openml.tasks.list_tasks(task_type_id=5, size=10) + # the expected outcome is that it doesn't crash. No assertions. def _check_task(self, task): self.assertEqual(type(task), dict) @@ -133,7 +133,6 @@ def assert_and_raise(*args, **kwargs): os.path.join(os.getcwd(), "tasks", "1", "tasks.xml") )) - def test_get_task_with_cache(self): openml.config.set_cache_directory(self.static_cache_dir) task = openml.tasks.get_task(1) From c878872710df5abc01efaeb4bf211644d7c34f41 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Wed, 29 Nov 2017 13:32:59 +0100 Subject: [PATCH 45/86] added unit test --- openml/runs/functions.py | 14 ++------ tests/test_runs/test_run_functions.py | 46 +++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 3196a5bdb..56168fd6b 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -681,7 +681,7 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): else: raise AttributeError('Run XML does not contain required (server) field: ', fieldname) - run = xmltodict.parse(xml)["oml:run"] + run = xmltodict.parse(xml, force_dict=['oml:file', 'oml:evaluation'])["oml:run"] run_id = obtain_field(run, 'oml:run_id', from_server, cast=int) uploader = obtain_field(run, 'oml:uploader', from_server, cast=int) uploader_name = obtain_field(run, 'oml:uploader_name', from_server) @@ -722,17 +722,9 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): else: output_data = run['oml:output_data'] if 'oml:file' in output_data: - if isinstance(output_data['oml:file'], dict): - # only one result.. probably due to an upload error - file_dict = output_data['oml:file'] - files[file_dict['oml:name']] = int(file_dict['oml:file_id']) - elif isinstance(output_data['oml:file'], list): - # multiple files, the normal case - for file_dict in output_data['oml:file']: + # multiple files, the normal case + for file_dict in output_data['oml:file']: files[file_dict['oml:name']] = int(file_dict['oml:file_id']) - else: - raise TypeError(type(output_data['oml:file'])) - if 'oml:evaluation' in output_data: # in normal cases there should be evaluations, but in case there # was an error these could be absent diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 9c5e1d1b3..5f2e00e84 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -1,4 +1,5 @@ import arff +import collections import json import random import time @@ -714,9 +715,49 @@ def test_run_with_classifiers_in_param_grid(self): def test__run_task_get_arffcontent(self): task = openml.tasks.get_task(7) - class_labels = task.class_labels + num_instances = 320 + num_folds = 1 + num_repeats = 1 + + clf = SGDClassifier(loss='log', random_state=1) + res = openml.runs.functions._run_model_on_fold(clf, task, 0, 0, 0, True) + + arff_datacontent, arff_tracecontent, user_defined_measures, model = res + # predictions + self.assertIsInstance(arff_datacontent, list) + # trace. SGD does not produce any + self.assertIsInstance(arff_tracecontent, list) + self.assertEquals(len(arff_tracecontent), 0) + + fold_evaluations = collections.defaultdict(lambda: collections.defaultdict(dict)) + for measure in user_defined_measures: + fold_evaluations[measure][0][0] = user_defined_measures[measure] + + self._check_fold_evaluations(fold_evaluations, num_repeats, num_folds) + + # 10 times 10 fold CV of 150 samples + self.assertEqual(len(arff_datacontent), num_instances * num_repeats) + for arff_line in arff_datacontent: + # check number columns + self.assertEqual(len(arff_line), 8) + # check repeat + self.assertGreaterEqual(arff_line[0], 0) + self.assertLessEqual(arff_line[0], num_repeats - 1) + # check fold + self.assertGreaterEqual(arff_line[1], 0) + self.assertLessEqual(arff_line[1], num_folds - 1) + # check row id + self.assertGreaterEqual(arff_line[2], 0) + self.assertLessEqual(arff_line[2], num_instances - 1) + # check confidences + self.assertAlmostEqual(sum(arff_line[4:6]), 1.0) + self.assertIn(arff_line[6], ['won', 'nowin']) + self.assertIn(arff_line[7], ['won', 'nowin']) + + def test__run_model_on_fold(self): + task = openml.tasks.get_task(11) num_instances = 3196 - num_folds = 10 + num_folds = 1 num_repeats = 1 clf = SGDClassifier(loss='log', random_state=1) @@ -748,6 +789,7 @@ def test__run_task_get_arffcontent(self): self.assertIn(arff_line[6], ['won', 'nowin']) self.assertIn(arff_line[7], ['won', 'nowin']) + def test__create_trace_from_arff(self): with open(self.static_cache_dir + '/misc/trace.arff', 'r') as arff_file: trace_arff = arff.load(arff_file) From 9f0ada9f19df355c59921bf604448998caf88874 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Wed, 29 Nov 2017 13:45:45 +0100 Subject: [PATCH 46/86] fix unittest (?) --- tests/test_runs/test_run_functions.py | 35 +++++++++++++-------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 5f2e00e84..64e631c99 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -714,24 +714,18 @@ def test_run_with_classifiers_in_param_grid(self): task=task, model=clf, avoid_duplicate_runs=False) def test__run_task_get_arffcontent(self): - task = openml.tasks.get_task(7) - num_instances = 320 - num_folds = 1 + task = openml.tasks.get_task(11) + num_instances = 3196 + num_folds = 10 num_repeats = 1 clf = SGDClassifier(loss='log', random_state=1) - res = openml.runs.functions._run_model_on_fold(clf, task, 0, 0, 0, True) - - arff_datacontent, arff_tracecontent, user_defined_measures, model = res + res = openml.runs.functions._run_task_get_arffcontent(clf, task) + arff_datacontent, arff_tracecontent, _, fold_evaluations, sample_evaluations = res # predictions self.assertIsInstance(arff_datacontent, list) # trace. SGD does not produce any - self.assertIsInstance(arff_tracecontent, list) - self.assertEquals(len(arff_tracecontent), 0) - - fold_evaluations = collections.defaultdict(lambda: collections.defaultdict(dict)) - for measure in user_defined_measures: - fold_evaluations[measure][0][0] = user_defined_measures[measure] + self.assertIsInstance(arff_tracecontent, type(None)) self._check_fold_evaluations(fold_evaluations, num_repeats, num_folds) @@ -755,18 +749,24 @@ def test__run_task_get_arffcontent(self): self.assertIn(arff_line[7], ['won', 'nowin']) def test__run_model_on_fold(self): - task = openml.tasks.get_task(11) - num_instances = 3196 + task = openml.tasks.get_task(7) + num_instances = 1054 num_folds = 1 num_repeats = 1 clf = SGDClassifier(loss='log', random_state=1) - res = openml.runs.functions._run_task_get_arffcontent(clf, task) - arff_datacontent, arff_tracecontent, _, fold_evaluations, sample_evaluations = res + res = openml.runs.functions._run_model_on_fold(clf, task, 0, 0, 0, True) + + arff_datacontent, arff_tracecontent, user_defined_measures, model = res # predictions self.assertIsInstance(arff_datacontent, list) # trace. SGD does not produce any - self.assertIsInstance(arff_tracecontent, type(None)) + self.assertIsInstance(arff_tracecontent, list) + self.assertEquals(len(arff_tracecontent), 0) + + fold_evaluations = collections.defaultdict(lambda: collections.defaultdict(dict)) + for measure in user_defined_measures: + fold_evaluations[measure][0][0] = user_defined_measures[measure] self._check_fold_evaluations(fold_evaluations, num_repeats, num_folds) @@ -789,7 +789,6 @@ def test__run_model_on_fold(self): self.assertIn(arff_line[6], ['won', 'nowin']) self.assertIn(arff_line[7], ['won', 'nowin']) - def test__create_trace_from_arff(self): with open(self.static_cache_dir + '/misc/trace.arff', 'r') as arff_file: trace_arff = arff.load(arff_file) From dc82bbf8c37efdbfc88560e63f80ae49541ac011 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Wed, 29 Nov 2017 13:47:23 +0100 Subject: [PATCH 47/86] fix unit test (!) --- tests/test_runs/test_run_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 64e631c99..1785fdc85 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -714,7 +714,7 @@ def test_run_with_classifiers_in_param_grid(self): task=task, model=clf, avoid_duplicate_runs=False) def test__run_task_get_arffcontent(self): - task = openml.tasks.get_task(11) + task = openml.tasks.get_task(7) num_instances = 3196 num_folds = 10 num_repeats = 1 @@ -750,7 +750,7 @@ def test__run_task_get_arffcontent(self): def test__run_model_on_fold(self): task = openml.tasks.get_task(7) - num_instances = 1054 + num_instances = 320 num_folds = 1 num_repeats = 1 From aa51d3de3ff8d2ab055eeb927b687b7ca2b227fb Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Wed, 29 Nov 2017 14:01:58 +0100 Subject: [PATCH 48/86] bugfix force_dict -> force_list --- openml/runs/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 56168fd6b..32c1bcbbe 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -681,7 +681,7 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): else: raise AttributeError('Run XML does not contain required (server) field: ', fieldname) - run = xmltodict.parse(xml, force_dict=['oml:file', 'oml:evaluation'])["oml:run"] + run = xmltodict.parse(xml, force_list=['oml:file', 'oml:evaluation'])["oml:run"] run_id = obtain_field(run, 'oml:run_id', from_server, cast=int) uploader = obtain_field(run, 'oml:uploader', from_server, cast=int) uploader_name = obtain_field(run, 'oml:uploader_name', from_server) From 08a39e00cb9fecc67e0cafb0b0a9432e9bd1b30b Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Wed, 29 Nov 2017 14:17:05 +0100 Subject: [PATCH 49/86] fix 2.7 bug --- tests/test_runs/test_run_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 1785fdc85..1049d223b 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -755,7 +755,8 @@ def test__run_model_on_fold(self): num_repeats = 1 clf = SGDClassifier(loss='log', random_state=1) - res = openml.runs.functions._run_model_on_fold(clf, task, 0, 0, 0, True) + can_measure_runtime = sys.version_info[:2] >= (3, 3) + res = openml.runs.functions._run_model_on_fold(clf, task, 0, 0, 0, can_measure_runtime) arff_datacontent, arff_tracecontent, user_defined_measures, model = res # predictions From 6147c4f48b6cf16142953f64bb45dc4ab3994150 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Mon, 18 Dec 2017 16:13:15 +0100 Subject: [PATCH 50/86] fixes #373 + unit test --- openml/runs/functions.py | 2 +- tests/test_runs/test_run_functions.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 32c1bcbbe..c95990946 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -594,7 +594,7 @@ def _extract_arfftrace_attributes(model): for key in model.cv_results_: if key.startswith('param_'): # supported types should include all types, including bool, int float - supported_types = (bool, int, float, six.string_types) + supported_types = (bool, int, float, six.string_types, tuple) if all(isinstance(i, supported_types) or i is None for i in model.cv_results_[key]): type = 'STRING' else: diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 1049d223b..4d14175ba 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -27,6 +27,7 @@ from sklearn.feature_selection import VarianceThreshold from sklearn.linear_model import LogisticRegression, SGDClassifier, \ LinearRegression +from sklearn.neural_network import MLPClassifier from sklearn.ensemble import RandomForestClassifier, BaggingClassifier from sklearn.svm import SVC, LinearSVC from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, \ @@ -614,13 +615,13 @@ def test__get_seeded_model_raises(self): self.assertRaises(ValueError, _get_seeded_model, model=clf, seed=42) def test__extract_arfftrace(self): - param_grid = {"max_depth": [3, None], - "max_features": [1, 2, 3, 4], - "bootstrap": [True, False], - "criterion": ["gini", "entropy"]} + param_grid = {"hidden_layer_sizes": [(5, 5), (10, 10), (20, 20)], + "activation" : ['identity', 'logistic', 'tanh', 'relu'], + "learning_rate_init": [0.1, 0.01, 0.001, 0.0001], + "max_iter": [10, 20, 40, 80]} num_iters = 10 task = openml.tasks.get_task(20) - clf = RandomizedSearchCV(RandomForestClassifier(), param_grid, num_iters) + clf = RandomizedSearchCV(MLPClassifier(), param_grid, num_iters) # just run the task train, _ = task.get_train_test_split_indices(0, 0) X, y = task.get_X_and_y() From fc9114607b79e9fc7587226e4f0fa61558864118 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Fri, 22 Dec 2017 15:30:18 +0100 Subject: [PATCH 51/86] data upload --- openml/datasets/dataset.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 4c98d2e64..85ef0cbcb 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -121,7 +121,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None, with open(self.data_pickle_file, "wb") as fh: pickle.dump((X, categorical, attribute_names), fh, -1) logger.debug("Saved dataset %d: %s to file %s" % - (self.dataset_id, self.name, self.data_pickle_file)) + (int(self.dataset_id or -1), self.name, self.data_pickle_file)) def push_tag(self, tag): """Annotates this data set with a tag on the server. @@ -446,7 +446,11 @@ def _to_xml(self): for prop in props: content = getattr(self, prop, None) if content is not None: - xml_dataset += "{1}\n".format(prop, content) + if isinstance(content, (list,set)): + for item in content: + xml_dataset += "{1}\n".format(prop, item) + else: + xml_dataset += "{1}\n".format(prop, content) xml_dataset += "" return xml_dataset From eb1b86936c0ec9eb59d30c8f0500697c0212f85c Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sat, 10 Feb 2018 16:47:48 +0100 Subject: [PATCH 52/86] adds tagging and untagging functions --- openml/utils.py | 50 ++++++++++++++++++- tests/test_datasets/test_dataset_functions.py | 11 ++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/openml/utils.py b/openml/utils.py index ea2bf2fa4..e15c31fa8 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -1,4 +1,6 @@ +import xmltodict import six +from ._api_calls import _perform_api_call def extract_xml_tags(xml_tag_name, node, allow_none=True): @@ -37,4 +39,50 @@ def extract_xml_tags(xml_tag_name, node, allow_none=True): return None else: raise ValueError("Could not find tag '%s' in node '%s'" % - (xml_tag_name, str(node))) \ No newline at end of file + (xml_tag_name, str(node))) + + +def _tag_entity(entity_type, entity_id, tag, untag=False): + """Abstract function that can be used as a partial for tagging entities + on OpenML + + Parameters + ---------- + entity_type : str + Name of the entity to tag (e.g., run, flow, data) + + entity_id : int + OpenML id of the entity + + tag : str + The tag + + untag : bool + Set to true if needed to untag, rather than tag + + Returns + ------- + tags : list + List of tags that the entity is still tagged with + """ + legal_entities = {'data', 'task', 'flow', 'setup', 'run'} + if entity_type not in legal_entities: + raise ValueError('Can\'t tag a %s' %entity_type) + + uri = '%s/tag' %entity_type + main_tag = 'oml:%s_tag' %entity_type + if untag: + uri = '%s/untag' %entity_type + main_tag = 'oml:%s_untag' %entity_type + + + post_variables = {'%s_id'%entity_type: entity_id, 'tag': tag} + result_xml = _perform_api_call(uri, post_variables) + + result = xmltodict.parse(result_xml, force_list={'oml:tag'})[main_tag] + + if 'oml:tag' in result: + return result['oml:tag'] + else: + # no tags, return empty list + return [] \ No newline at end of file diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 6bbe6525f..8ccc91861 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -8,6 +8,7 @@ else: import mock +import random import six import scipy.sparse @@ -15,6 +16,7 @@ from openml import OpenMLDataset from openml.exceptions import OpenMLCacheException, PyOpenMLError from openml.testing import TestBase +from openml.utils import _tag_entity from openml.datasets.functions import (_get_cached_dataset, _get_cached_dataset_features, @@ -105,6 +107,15 @@ def _check_dataset(self, dataset): self.assertIn(dataset['status'], ['in_preparation', 'active', 'deactivated']) + def test_tag_untag_dataset(self): + tag = 'test_tag_%d' %random.randint(1, 1000000) + all_tags = _tag_entity('data', 1, tag) + self.assertTrue(tag in all_tags) + all_tags = _tag_entity('data', 1, tag, untag=True) + self.assertTrue(tag not in all_tags) + + + def test_list_datasets(self): # We can only perform a smoke test here because we test on dynamic # data from the internet... From 5de61016126b32a3cbb8c0012dbed97764be858f Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sat, 10 Feb 2018 17:48:15 +0100 Subject: [PATCH 53/86] specified the error type that we expect in unit test --- tests/test_runs/test_run_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 1049d223b..cd51d4144 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -518,7 +518,10 @@ def test_get_run_trace(self): run = run.publish() self._wait_for_processed_run(run.run_id, 200) run_id = run.run_id - except openml.exceptions.PyOpenMLError: + except openml.exceptions.PyOpenMLError as e: + if 'Run already exists in server' not in e.message: + # in this case the error was not the one we expected + raise e # run was already flow = openml.flows.sklearn_to_flow(clf) flow_exists = openml.flows.flow_exists(flow.name, flow.external_version) From 7fdb403579619a09fefaf07cbd998dbca29e8781 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sat, 10 Feb 2018 18:19:17 +0100 Subject: [PATCH 54/86] tiny enhencement --- openml/datasets/dataset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 799ed9fb7..d9ac3456b 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -92,7 +92,10 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None, self.qualities = {} for idx, xmlquality in enumerate(qualities['oml:quality']): name = xmlquality['oml:name'] - value = xmlquality['oml:value'] + if 'oml:value' in xmlquality: + value = xmlquality['oml:value'] + else: + value = None self.qualities[name] = value if data_file is not None: From de999ad0f65c13a7a0f9e449bddfaddd14341000 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sat, 10 Feb 2018 20:10:45 +0100 Subject: [PATCH 55/86] added setup caching --- openml/runs/functions.py | 2 +- openml/setups/functions.py | 36 ++++++++++++++++++++--- tests/test_setups/test_setup_functions.py | 12 +++++++- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 73d039464..e527c1d2d 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -620,7 +620,7 @@ def _create_run_from_xml(xml): if 'oml:output_data' not in run: raise ValueError('Run does not contain output_data (OpenML server error?)') - if 'oml:file' in 'oml:output_data': + if 'oml:file' in run['oml:output_data']: if isinstance(run['oml:output_data']['oml:file'], dict): # only one result.. probably due to an upload error file_dict = run['oml:output_data']['oml:file'] diff --git a/openml/setups/functions.py b/openml/setups/functions.py index a221e2aec..cb212950a 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -1,8 +1,11 @@ from collections import OrderedDict +import io import openml +import os import xmltodict +from .. import config from .setup import OpenMLSetup, OpenMLParameter from openml.flows import flow_exists @@ -54,8 +57,23 @@ def setup_exists(flow, model=None): return False +def _get_cached_setup(setup_id): + """Load a run from the cache.""" + cache_dir = config.get_cache_directory() + setup_cache_dir = os.path.join(cache_dir, "setups") + try: + setup_file = os.path.join(setup_cache_dir, "setup_%d.xml" % int(setup_id)) + with io.open(setup_file, encoding='utf8') as fh: + setup_xml = xmltodict.parse(fh.read()) + setup = _create_setup_from_xml(setup_xml) + return setup + + except (OSError, IOError): + raise openml.exceptions.OpenMLCacheException("Setup file for setup id %d not cached" % setup_id) + + def get_setup(setup_id): - ''' + """ Downloads the setup (configuration) description from OpenML and returns a structured object @@ -68,9 +86,18 @@ def get_setup(setup_id): ------- OpenMLSetup an initialized openml setup object - ''' - result = openml._api_calls._perform_api_call('/setup/%d' %setup_id) - result_dict = xmltodict.parse(result) + """ + run_file = os.path.join(config.get_cache_directory(), "setups", "setup_%d.xml" % setup_id) + + try: + return _get_cached_setup(setup_id) + + except (openml.exceptions.OpenMLCacheException): + setup_xml = openml._api_calls._perform_api_call('/setup/%d' % setup_id) + with io.open(run_file, "w", encoding='utf8') as fh: + fh.write(setup_xml) + + result_dict = xmltodict.parse(setup_xml) return _create_setup_from_xml(result_dict) @@ -217,6 +244,7 @@ def _to_dict(flow_id, openml_parameter_settings): return xml + def _create_setup_from_xml(result_dict): ''' Turns an API xml result into a OpenMLSetup object diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 88e98708f..da81ec53d 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -140,4 +140,14 @@ def test_setuplist_offset(self): all = set(setups.keys()).union(setups2.keys()) - self.assertEqual(len(all), size * 2) \ No newline at end of file + self.assertEqual(len(all), size * 2) + + def test_get_cached_setup(self): + openml.config.set_cache_directory(self.static_cache_dir) + openml.setups.functions._get_cached_setup(1) + + + def test_get_uncached_setup(self): + openml.config.set_cache_directory(self.static_cache_dir) + with self.assertRaises(openml.exceptions.OpenMLCacheException): + openml.setups.functions._get_cached_setup(10) \ No newline at end of file From 8726314edf41cca2034cf2dc5c3282a89e0aa465 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sun, 11 Feb 2018 11:36:37 +0100 Subject: [PATCH 56/86] fix unit tests, changed directory structure to be consistent with other types --- openml/runs/functions.py | 9 +++++++-- openml/setups/functions.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index e527c1d2d..75d701864 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -555,8 +555,13 @@ def get_run(run_id): run : OpenMLRun Run corresponding to ID, fetched from the server. """ - run_file = os.path.join(config.get_cache_directory(), "runs", - "run_%d.xml" % run_id) + run_dir = config.get_cache_directory(), "runs", run_id + run_file = os.path.join(run_dir, "description.xml") + + try: + os.makedirs(run_dir) + except FileExistsError: + pass try: return _get_cached_run(run_id) diff --git a/openml/setups/functions.py b/openml/setups/functions.py index cb212950a..613b84ac6 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -87,14 +87,20 @@ def get_setup(setup_id): OpenMLSetup an initialized openml setup object """ - run_file = os.path.join(config.get_cache_directory(), "setups", "setup_%d.xml" % setup_id) + setup_dir = config.get_cache_directory(), "runs", setup_id + setup_file = os.path.join(setup_dir, "description.xml") + + try: + os.makedirs(setup_dir) + except FileExistsError: + pass try: return _get_cached_setup(setup_id) except (openml.exceptions.OpenMLCacheException): setup_xml = openml._api_calls._perform_api_call('/setup/%d' % setup_id) - with io.open(run_file, "w", encoding='utf8') as fh: + with io.open(setup_file, "w", encoding='utf8') as fh: fh.write(setup_xml) result_dict = xmltodict.parse(setup_xml) From 264677adfe53a589256142e2e3f38a241fea2c63 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sun, 11 Feb 2018 11:39:54 +0100 Subject: [PATCH 57/86] fixes unit tests --- openml/runs/functions.py | 2 +- openml/setups/functions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 75d701864..114e8c662 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -555,7 +555,7 @@ def get_run(run_id): run : OpenMLRun Run corresponding to ID, fetched from the server. """ - run_dir = config.get_cache_directory(), "runs", run_id + run_dir = os.path.join(config.get_cache_directory(), "runs", str(run_id)) run_file = os.path.join(run_dir, "description.xml") try: diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 613b84ac6..d2a407abd 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -87,7 +87,7 @@ def get_setup(setup_id): OpenMLSetup an initialized openml setup object """ - setup_dir = config.get_cache_directory(), "runs", setup_id + setup_dir = os.path.join(config.get_cache_directory(), "setups", str(setup_id)) setup_file = os.path.join(setup_dir, "description.xml") try: From bc25db8b74ea49b9f1b283b7a0ccc4823717fdf8 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sun, 11 Feb 2018 12:12:59 +0100 Subject: [PATCH 58/86] added fake cache, added unit test, fixed cache location --- openml/runs/functions.py | 8 +- openml/setups/functions.py | 4 +- tests/files/runs/1/description.xml | 140 ++++ tests/files/setups/1/description.xml | 977 ++++++++++++++++++++++++++ tests/test_runs/test_run_functions.py | 9 + 5 files changed, 1131 insertions(+), 7 deletions(-) create mode 100644 tests/files/runs/1/description.xml create mode 100644 tests/files/setups/1/description.xml diff --git a/openml/runs/functions.py b/openml/runs/functions.py index e68be4026..2d8a55595 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -699,7 +699,6 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): else: task_evaluation_measure = None - flow_id = int(run['oml:flow_id']) flow_name = obtain_field(run, 'oml:flow_name', from_server) setup_id = obtain_field(run, 'oml:setup_id', from_server, cast=int) @@ -714,7 +713,7 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): parameters[key] = value if 'oml:input_data' in run: - dataset_id = int(run['oml:input_data']['oml:dataset']['oml:did']) + dataset_id = int(run['oml:input_data']['oml:dataset'][0]['oml:did']) elif not from_server: dataset_id = None @@ -877,10 +876,9 @@ def _create_trace_from_arff(arff_obj): def _get_cached_run(run_id): """Load a run from the cache.""" cache_dir = config.get_cache_directory() - run_cache_dir = os.path.join(cache_dir, "runs") + run_cache_dir = os.path.join(cache_dir, "runs", str(run_id)) try: - run_file = os.path.join(run_cache_dir, - "run_%d.xml" % int(run_id)) + run_file = os.path.join(run_cache_dir, "description.xml") with io.open(run_file, encoding='utf8') as fh: run = _create_run_from_xml(xml=fh.read()) return run diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 67808f832..7611a0633 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -60,9 +60,9 @@ def setup_exists(flow, model=None): def _get_cached_setup(setup_id): """Load a run from the cache.""" cache_dir = config.get_cache_directory() - setup_cache_dir = os.path.join(cache_dir, "setups") + setup_cache_dir = os.path.join(cache_dir, "setups", str(setup_id)) try: - setup_file = os.path.join(setup_cache_dir, "setup_%d.xml" % int(setup_id)) + setup_file = os.path.join(setup_cache_dir, "description.xml" % int(setup_id)) with io.open(setup_file, encoding='utf8') as fh: setup_xml = xmltodict.parse(fh.read()) setup = _create_setup_from_xml(setup_xml) diff --git a/tests/files/runs/1/description.xml b/tests/files/runs/1/description.xml new file mode 100644 index 000000000..f0669eb37 --- /dev/null +++ b/tests/files/runs/1/description.xml @@ -0,0 +1,140 @@ + + 1 + 1 + Jan van Rijn + 68 + Learning Curve + predictive_accuracy + 61 + weka.REPTree(1) + 6 + weka.classifiers.trees.REPTree -- -M 2 -V 0.001 -N 3 -S 1 -L -1 -I 0.0 + + I + 0.0 + + + L + -1 + + + M + 2 + + + N + 3 + + + S + 1 + + + V + 0.001 + + curves + test_base_tagOMLObject_2 + weka + weka_3.7.12-SNAPSHOT + + + 2 + anneal + https://www.openml.org/data/download/1666876/phpFsFYVN + + + 9 + autos + https://www.openml.org/data/download/9/dataset_9_autos.arff + + + 54 + vehicle + https://www.openml.org/data/download/54/dataset_54_vehicle.arff + + + + + -1 + 63 + description + https://www.openml.org/data/download/63/weka_generated_run5258986433356798974.xml + + + -1 + 313413 + model_readable + https://www.openml.org/data/download/313413/WekaModel_weka.classifiers.bayes.AveragedNDependenceEstimators.A1DE2474025000319897700.model + + + -1 + 622143 + model_serialized + https://www.openml.org/data/download/622143/WekaSerialized_weka.classifiers.bayes.AveragedNDependenceEstimators.A1DE3739675682024987582.model + + + -1 + 64 + predictions + https://www.openml.org/data/download/64/weka_generated_predictions5823074444642592781.arff + + + area_under_roc_curve + 0.839359 [0.0,0.99113,0.898048,0.874862,0.791282,0.807343,0.820674] + + average_cost + 0 + + f_measure + 0.600026 [0,0,0.711934,0.735714,0.601363,0.435678,0.430913] + + kappa + 0.491678 + + kb_relative_information_score + 1063.298606 + + mean_absolute_error + 0.127077 + + mean_prior_absolute_error + 0.220919 + + number_of_instances + 2050 [0,30,220,670,540,320,270] + + os_information + [ Oracle Corporation, 1.7.0_51, amd64, Linux, 3.7.10-1.28-desktop ] + + precision + 0.599589 [0,0,0.650376,0.705479,0.556782,0.48289,0.585987] + + predictive_accuracy + 0.614634 + + prior_entropy + 2.326811 + + recall + 0.614634 [0,0,0.786364,0.768657,0.653704,0.396875,0.340741] + + relative_absolute_error + 0.575218 + + root_mean_prior_squared_error + 0.331758 + + root_mean_squared_error + 0.280656 + + root_relative_squared_error + 0.845964 + + scimark_benchmark + 1973.4091512218106 [ 1262.1133708514062, 1630.9393838458018, 932.0675956790141, 1719.5408190761134, 4322.384586656718 ] + + total_cost + 0 + + diff --git a/tests/files/setups/1/description.xml b/tests/files/setups/1/description.xml new file mode 100644 index 000000000..f918b0573 --- /dev/null +++ b/tests/files/setups/1/description.xml @@ -0,0 +1,977 @@ + + 1 + 56 + + 228 + 392 + weka.A1DE(2)_F + F + option + 1 + 1 + + + 229 + 392 + weka.A1DE(2)_M + M + option + 1.0 + 1.0 + + + 295 + 376 + weka.AdaBoostM1_DecisionStump(2)_I + I + option + 10 + 10 + + + 296 + 376 + weka.AdaBoostM1_DecisionStump(2)_P + P + option + 100 + 100 + + + 298 + 376 + weka.AdaBoostM1_DecisionStump(2)_S + S + option + 1 + 1 + + + 299 + 376 + weka.AdaBoostM1_DecisionStump(2)_W + W + baselearner + weka.classifiers.trees.DecisionStump + weka.classifiers.trees.DecisionStump + + + 334 + 404 + weka.AdaBoostM1_IBk(2)_I + I + option + 10 + 20 + + + 335 + 404 + weka.AdaBoostM1_IBk(2)_P + P + option + 100 + 100 + + + 337 + 404 + weka.AdaBoostM1_IBk(2)_S + S + option + 1 + 1 + + + 338 + 404 + weka.AdaBoostM1_IBk(2)_W + W + baselearner + weka.classifiers.lazy.IBk + weka.classifiers.lazy.IBk + + + 2883 + 530 + weka.Bagging_NaiveBayes(2)_I + I + option + 10 + 80 + + + 2888 + 530 + weka.Bagging_NaiveBayes(2)_P + P + option + 100 + 100 + + + 2890 + 530 + weka.Bagging_NaiveBayes(2)_S + S + option + 1 + 1 + + + 2892 + 530 + weka.Bagging_NaiveBayes(2)_W + W + baselearner + weka.classifiers.bayes.NaiveBayes + weka.classifiers.bayes.NaiveBayes + + + 2893 + 530 + weka.Bagging_NaiveBayes(2)_num-slots + num-slots + option + 1 + 1 + + + 2909 + 531 + weka.Bagging_OneR(2)_I + I + option + 10 + 160 + + + 2914 + 531 + weka.Bagging_OneR(2)_P + P + option + 100 + 100 + + + 2916 + 531 + weka.Bagging_OneR(2)_S + S + option + 1 + 1 + + + 2918 + 531 + weka.Bagging_OneR(2)_W + W + baselearner + weka.classifiers.rules.OneR + weka.classifiers.rules.OneR + + + 2919 + 531 + weka.Bagging_OneR(2)_num-slots + num-slots + option + 1 + 1 + + + 3132 + 397 + weka.BayesianLogisticRegression(2)_D + D + flag + true + true + + + 3133 + 397 + weka.BayesianLogisticRegression(2)_F + F + option + 2 + 2 + + + 3134 + 397 + weka.BayesianLogisticRegression(2)_H + H + option + 1 + 1 + + + 3135 + 397 + weka.BayesianLogisticRegression(2)_I + I + option + 100 + 100 + + + 3136 + 397 + weka.BayesianLogisticRegression(2)_N + N + flag + true + true + + + 3137 + 397 + weka.BayesianLogisticRegression(2)_P + P + option + 1 + 1 + + + 3138 + 397 + weka.BayesianLogisticRegression(2)_R + R + option + R:0.01-316,3.16 + R:0.01-316,3.16 + + + 3139 + 397 + weka.BayesianLogisticRegression(2)_S + S + option + 0.5 + 0.5 + + + 3140 + 397 + weka.BayesianLogisticRegression(2)_Tl + Tl + option + 5.0E-4 + 5.0E-4 + + + 3141 + 397 + weka.BayesianLogisticRegression(2)_V + V + option + 0.27 + 0.27 + + + 3142 + 397 + weka.BayesianLogisticRegression(2)_seed + seed + option + 1 + 1 + + + 3191 + 441 + weka.ComplementNaiveBayes(2)_S + S + option + 1.0 + 1.0 + + + 3335 + 401 + weka.GaussianProcesses_PolyKernel(3)_L + L + option + 1.0 + 1.0 + + + 3336 + 401 + weka.GaussianProcesses_PolyKernel(3)_N + N + option + 0 + 0 + + + 3409 + 389 + weka.IBk(2)_A + A + flag + true + true + + + 3413 + 389 + weka.IBk(2)_K + K + option + 1 + 1 + + + 3414 + 389 + weka.IBk(2)_W + W + option + 0 + 0 + + + 3951 + 375 + weka.OneR(3)_B + B + option + 6 + 6 + + + 4028 + 386 + weka.PolyKernel(4)_C + C + option + 250007 + 250007 + + + 4029 + 386 + weka.PolyKernel(4)_E + E + option + 1.0 + 1.0 + + + 5431 + 564 + weka.GaussianProcesses_NormalizedPolyKernel(2)_L + L + option + 1.0 + 1.0 + + + 5432 + 564 + weka.GaussianProcesses_NormalizedPolyKernel(2)_N + N + option + 0 + 0 + + + 5433 + 564 + weka.GaussianProcesses_NormalizedPolyKernel(2)_K + K + kernel + weka.classifiers.functions.supportVector.NormalizedPolyKernel + weka.classifiers.functions.supportVector.NormalizedPolyKernel + + + 5439 + 565 + weka.NormalizedPolyKernel(2)_E + E + option + 2.0 + 2.0 + + + 5441 + 565 + weka.NormalizedPolyKernel(2)_C + C + option + 250007 + 250007 + + + 5566 + 582 + weka.CVParameterSelection_ZeroR(2)_X + X + option + 10 + 10 + + + 5568 + 582 + weka.CVParameterSelection_ZeroR(2)_S + S + option + 1 + 1 + + + 5569 + 582 + weka.CVParameterSelection_ZeroR(2)_W + W + baselearner + weka.classifiers.rules.ZeroR + weka.classifiers.rules.ZeroR + + + 6514 + 675 + weka.J48(13)_C + C + option + 0.25 + 0.25 + + + 6515 + 675 + weka.J48(13)_M + M + option + 2 + 2 + + + 9551 + 1068 + weka.J48(28)_C + C + option + 0.25 + 0.25 + + + 9552 + 1068 + weka.J48(28)_M + M + option + 2 + 2 + + + 9601 + 1077 + weka.REPTree(9)_M + M + option + 2 + 2 + + + 9602 + 1077 + weka.REPTree(9)_V + V + option + 0.001 + 0.001 + + + 9603 + 1077 + weka.REPTree(9)_N + N + option + 3 + 3 + + + 9604 + 1077 + weka.REPTree(9)_S + S + option + 1 + 1 + + + 9606 + 1077 + weka.REPTree(9)_L + L + option + -1 + -1 + + + 9607 + 1077 + weka.REPTree(9)_I + I + option + 0.0 + 0.0 + + + 9611 + 1078 + weka.RandomTree(10)_K + K + option + 0 + 0 + + + 9612 + 1078 + weka.RandomTree(10)_M + M + option + 1.0 + 1.0 + + + 9613 + 1078 + weka.RandomTree(10)_V + V + option + 0.001 + 0.001 + + + 9614 + 1078 + weka.RandomTree(10)_S + S + option + 1 + 1 + + + 9620 + 1079 + weka.RandomForest(5)_I + I + option + 100 + 100 + + + 9621 + 1079 + weka.RandomForest(5)_K + K + option + 0 + 0 + + + 9622 + 1079 + weka.RandomForest(5)_S + S + option + 1 + 1 + + + 9626 + 1079 + weka.RandomForest(5)_num-slots + num-slots + option + 1 + 1 + + + 9873 + 1116 + weka.MultilayerPerceptronCS(1)_H + H + option + a + a + + + 10227 + 1165 + weka.A1DE(4)_F + F + option + 1 + 1 + + + 10228 + 1165 + weka.A1DE(4)_M + M + option + 1.0 + 1.0 + + + 10246 + 1168 + weka.BayesNet_K2(6)_D + D + flag + true + true + + + 10248 + 1168 + weka.BayesNet_K2(6)_Q + Q + baselearner + weka.classifiers.bayes.net.search.local.K2 + weka.classifiers.bayes.net.search.local.K2 + + + 10259 + 1169 + weka.K2(5)_P + P + option + 1 + 1 + + + 10262 + 1169 + weka.K2(5)_S + S + option + BAYES + BAYES + + + 10268 + 1172 + weka.LibSVM(2)_S + S + option + 0 + 0 + + + 10269 + 1172 + weka.LibSVM(2)_K + K + option + 2 + 2 + + + 10270 + 1172 + weka.LibSVM(2)_D + D + option + 3 + 3 + + + 10271 + 1172 + weka.LibSVM(2)_G + G + option + 0.0 + 0.0 + + + 10272 + 1172 + weka.LibSVM(2)_R + R + option + 0.0 + 0.0 + + + 10273 + 1172 + weka.LibSVM(2)_C + C + option + 1.0 + 1.0 + + + 10274 + 1172 + weka.LibSVM(2)_N + N + option + 0.5 + 0.5 + + + 10278 + 1172 + weka.LibSVM(2)_P + P + option + 0.1 + 0.1 + + + 10279 + 1172 + weka.LibSVM(2)_M + M + option + 40.0 + 40.0 + + + 10280 + 1172 + weka.LibSVM(2)_E + E + option + 0.001 + 0.001 + + + 10284 + 1172 + weka.LibSVM(2)_model + model + option + /Users/joa/Downloads/weka-3-7-12 + /Users/joa/Downloads/weka-3-7-12 + + + 10285 + 1172 + weka.LibSVM(2)_seed + seed + option + 1 + 1 + + + 10290 + 1174 + weka.KernelLogisticRegression_RBFKernel(1)_S + S + option + 1 + 1 + + + 10293 + 1174 + weka.KernelLogisticRegression_RBFKernel(1)_K + K + kernel + weka.classifiers.functions.supportVector.RBFKernel + weka.classifiers.functions.supportVector.RBFKernel + + + 10294 + 1174 + weka.KernelLogisticRegression_RBFKernel(1)_L + L + option + 0.01 + 0.01 + + + 10296 + 1174 + weka.KernelLogisticRegression_RBFKernel(1)_P + P + option + 1 + 1 + + + 10297 + 1174 + weka.KernelLogisticRegression_RBFKernel(1)_E + E + option + 1 + 1 + + + 10300 + 1175 + weka.RBFKernel(3)_G + G + option + 0.01 + 0.01 + + + 10301 + 1175 + weka.RBFKernel(3)_C + C + option + 250007 + 250007 + + + 10385 + 1185 + weka.Bagging_REPTree(8)_P + P + option + 100 + 100 + + + 10388 + 1185 + weka.Bagging_REPTree(8)_S + S + option + 1 + 1 + + + 10389 + 1185 + weka.Bagging_REPTree(8)_num-slots + num-slots + option + 1 + 1 + + + 10390 + 1185 + weka.Bagging_REPTree(8)_I + I + option + 10 + 10 + + + 10391 + 1185 + weka.Bagging_REPTree(8)_W + W + baselearner + weka.classifiers.trees.REPTree + weka.classifiers.trees.REPTree + + + 10547 + 1199 + weka.ADTree(4)_B + B + option + 10 + 10 + + + 10548 + 1199 + weka.ADTree(4)_E + E + option + -3 + -3 + + + 10550 + 1199 + weka.ADTree(4)_S + S + option + 1 + 1 + + + 10553 + 1200 + weka.LADTree(4)_B + B + option + 10 + 10 + + + 10749 + 1244 + weka.FilteredClassifier_Discretize_J48(4)_F + F + kernel + weka.filters.supervised.attribute.Discretize + weka.filters.supervised.attribute.Discretize + + + 10750 + 1244 + weka.FilteredClassifier_Discretize_J48(4)_W + W + baselearner + weka.classifiers.trees.J48 + weka.classifiers.trees.J48 + + + 10772 + 1245 + weka.Discretize(3)_R + R + option + first-last + first-last + + + 10778 + 1245 + weka.Discretize(3)_precision + precision + option + 6 + 6 + + + 14254 + 1717 + weka.RandomForest(5)_K + K + option + 0 + 0 + + + 14256 + 1717 + weka.RandomForest(5)_S + S + option + 1 + 1 + + + 14258 + 1717 + weka.RandomForest(5)_num-slots + num-slots + option + 1 + 1 + + + diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 1049d223b..7b70410cb 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -968,3 +968,12 @@ def test_predict_proba_hardclassifier(self): predictionsB = np.array(arff_content2)[:, -2:] np.testing.assert_array_equal(predictionsA, predictionsB) + + def test_get_cached_run(self): + openml.config.set_cache_directory(self.static_cache_dir) + openml.runs.functions._get_cached_run(1) + + def test_get_uncached_run(self): + openml.config.set_cache_directory(self.static_cache_dir) + with self.assertRaises(openml.exceptions.OpenMLCacheException): + openml.runs.functions._get_cached_run(10) \ No newline at end of file From 2efb09418eaedb51a047607b8f53cbb40268a4e7 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sun, 11 Feb 2018 12:25:28 +0100 Subject: [PATCH 59/86] changed cache files for setup and run, bugfix syntax and list indexing --- openml/runs/functions.py | 2 +- openml/setups/functions.py | 2 +- tests/files/runs/1/description.xml | 651 ++++++++++++++++-- tests/files/setups/1/description.xml | 972 +-------------------------- 4 files changed, 589 insertions(+), 1038 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 2d8a55595..66ab879e6 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -713,7 +713,7 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): parameters[key] = value if 'oml:input_data' in run: - dataset_id = int(run['oml:input_data']['oml:dataset'][0]['oml:did']) + dataset_id = int(run['oml:input_data']['oml:dataset']['oml:did']) elif not from_server: dataset_id = None diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 7611a0633..492129af2 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -62,7 +62,7 @@ def _get_cached_setup(setup_id): cache_dir = config.get_cache_directory() setup_cache_dir = os.path.join(cache_dir, "setups", str(setup_id)) try: - setup_file = os.path.join(setup_cache_dir, "description.xml" % int(setup_id)) + setup_file = os.path.join(setup_cache_dir, "description.xml") with io.open(setup_file, encoding='utf8') as fh: setup_xml = xmltodict.parse(fh.read()) setup = _create_setup_from_xml(setup_xml) diff --git a/tests/files/runs/1/description.xml b/tests/files/runs/1/description.xml index f0669eb37..92e9bcb98 100644 --- a/tests/files/runs/1/description.xml +++ b/tests/files/runs/1/description.xml @@ -1,140 +1,645 @@ - 1 + 100 1 Jan van Rijn - 68 - Learning Curve - predictive_accuracy - 61 - weka.REPTree(1) - 6 - weka.classifiers.trees.REPTree -- -M 2 -V 0.001 -N 3 -S 1 -L -1 -I 0.0 + 28 + Supervised Classification + 67 + weka.BayesNet_K2(1) + 12 + weka.classifiers.bayes.BayesNet -- -D -Q weka.classifiers.bayes.net.search.local.K2 -- -P 1 -S BAYES -E weka.classifiers.bayes.net.estimate.SimpleEstimator -- -A 0.5 - I - 0.0 + D + true - L - -1 + Q + weka.classifiers.bayes.net.search.local.K2 - M - 2 - - - N - 3 - - - S + P 1 - V - 0.001 + S + BAYES - curves - test_base_tagOMLObject_2 - weka - weka_3.7.12-SNAPSHOT - + - 2 - anneal - https://www.openml.org/data/download/1666876/phpFsFYVN - - - 9 - autos - https://www.openml.org/data/download/9/dataset_9_autos.arff - - - 54 - vehicle - https://www.openml.org/data/download/54/dataset_54_vehicle.arff + 28 + optdigits + https://www.openml.org/data/download/28/dataset_28_optdigits.arff -1 - 63 + 261 description - https://www.openml.org/data/download/63/weka_generated_run5258986433356798974.xml - - - -1 - 313413 - model_readable - https://www.openml.org/data/download/313413/WekaModel_weka.classifiers.bayes.AveragedNDependenceEstimators.A1DE2474025000319897700.model + https://www.openml.org/data/download/261/weka_generated_run935374685998857626.xml -1 - 622143 - model_serialized - https://www.openml.org/data/download/622143/WekaSerialized_weka.classifiers.bayes.AveragedNDependenceEstimators.A1DE3739675682024987582.model - - - -1 - 64 + 262 predictions - https://www.openml.org/data/download/64/weka_generated_predictions5823074444642592781.arff + https://www.openml.org/data/download/262/weka_generated_predictions576954524972002741.arff area_under_roc_curve - 0.839359 [0.0,0.99113,0.898048,0.874862,0.791282,0.807343,0.820674] + 0.990288 [0.99724,0.989212,0.992776,0.994279,0.980578,0.98649,0.99422,0.99727,0.994858,0.976143] average_cost 0 f_measure - 0.600026 [0,0,0.711934,0.735714,0.601363,0.435678,0.430913] + 0.922723 [0.989091,0.898857,0.935041,0.92431,0.927944,0.918156,0.980322,0.933219,0.895018,0.826531] kappa - 0.491678 + 0.913601 kb_relative_information_score - 1063.298606 + 5181.417432 mean_absolute_error - 0.127077 + 0.016374 mean_prior_absolute_error - 0.220919 + 0.179997 number_of_instances - 2050 [0,30,220,670,540,320,270] + 5620 [554,571,557,572,568,558,558,566,554,562] os_information [ Oracle Corporation, 1.7.0_51, amd64, Linux, 3.7.10-1.28-desktop ] precision - 0.599589 [0,0,0.650376,0.705479,0.556782,0.48289,0.585987] + 0.924345 [0.996337,0.902827,0.953358,0.941924,0.926316,0.966337,0.978571,0.905316,0.882456,0.791531] predictive_accuracy - 0.614634 + 0.922242 prior_entropy - 2.326811 + 3.321833 recall - 0.614634 [0,0,0.786364,0.768657,0.653704,0.396875,0.340741] + 0.922242 [0.981949,0.894921,0.917415,0.907343,0.929577,0.874552,0.982079,0.962898,0.907942,0.864769] relative_absolute_error - 0.575218 + 0.090968 root_mean_prior_squared_error - 0.331758 + 0.299998 root_mean_squared_error - 0.280656 + 0.117387 root_relative_squared_error - 0.845964 + 0.391293 scimark_benchmark - 1973.4091512218106 [ 1262.1133708514062, 1630.9393838458018, 932.0675956790141, 1719.5408190761134, 4322.384586656718 ] + 1969.9216824070186 [ 1241.8943613564243, 1575.5968355392279, 906.1111964820476, 1675.0415998938465, 4450.964418763549 ] total_cost + 0 + + area_under_roc_curve + 0.993338 [1,0.987789,0.9958,0.998255,0.988484,0.997318,0.989483,0.999259,0.99749,0.979708] + + area_under_roc_curve + 0.990543 [0.999928,0.977523,0.996012,0.995758,0.970974,0.978296,0.999929,0.998447,0.995804,0.993418] + + area_under_roc_curve + 0.990071 [0.990873,0.996839,0.994512,0.994268,0.988327,0.989448,0.999965,0.997881,0.994549,0.95384] + + area_under_roc_curve + 0.988339 [1,0.983081,0.979249,0.989456,0.989699,0.976937,1,0.999375,0.996199,0.969597] + + area_under_roc_curve + 0.993133 [1,0.992201,0.988848,1,0.998749,0.996288,0.981719,0.998506,0.996378,0.978826] + + area_under_roc_curve + 0.990209 [0.990837,0.997777,0.9982,0.997151,0.9786,0.985602,0.99086,0.990412,0.99125,0.981449] + + area_under_roc_curve + 0.989645 [0.990613,0.998558,0.999014,0.985044,0.980059,0.986784,0.990335,0.996161,0.998094,0.971944] + + area_under_roc_curve + 0.987188 [0.999929,0.974848,0.997418,0.995275,0.960292,0.969438,0.999928,0.997533,0.99213,0.985866] + + area_under_roc_curve + 0.987588 [1,0.987189,0.989098,0.988067,0.96208,0.98696,1,0.996206,0.99213,0.974185] + + area_under_roc_curve + 0.993119 [1,0.996648,0.989977,0.998089,0.990401,0.998412,0.989801,0.998765,0.994636,0.974308] + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + average_cost + 0 + + build_cpu_time + 0.058 + + build_cpu_time + 0.058 + + build_cpu_time + 0.055 + + build_cpu_time + 0.054 + + build_cpu_time + 0.052 + + build_cpu_time + 0.052 + + build_cpu_time + 0.051 + + build_cpu_time + 0.054 + + build_cpu_time + 0.052 + + build_cpu_time + 0.052 + + f_measure + 0.922001 [1,0.894737,0.954128,0.910714,0.902655,0.953271,0.947368,0.932203,0.902655,0.824561] + + f_measure + 0.933066 [0.990826,0.93578,0.929825,0.929825,0.921739,0.914286,0.99115,0.954955,0.910714,0.852459] + + f_measure + 0.921167 [0.990826,0.900901,0.915888,0.912281,0.932203,0.93578,0.990991,0.933333,0.923077,0.777778] + + f_measure + 0.92075 [0.990826,0.854701,0.914286,0.944444,0.932203,0.924528,1,0.957265,0.87931,0.810345] + + f_measure + 0.928674 [0.981481,0.882883,0.928571,0.974359,0.949153,0.915888,0.963636,0.966102,0.915888,0.810345] + + f_measure + 0.913056 [0.981481,0.907563,0.934579,0.912281,0.928571,0.836735,0.973451,0.973913,0.839286,0.84127] + + f_measure + 0.931169 [0.972973,0.93913,0.971963,0.914286,0.923077,0.910714,0.972477,0.905983,0.927273,0.876033] + + f_measure + 0.909739 [0.99115,0.841121,0.936937,0.928571,0.899083,0.914286,0.982143,0.898305,0.892857,0.816] + + f_measure + 0.906202 [0.990991,0.898305,0.915888,0.9,0.915888,0.903846,1,0.883333,0.864865,0.789474] + + f_measure + 0.939667 [1,0.931034,0.947368,0.915888,0.972973,0.963636,0.982143,0.929825,0.894737,0.859649] + + kappa + 0.91301 + + kappa + 0.924872 + + kappa + 0.913007 + + kappa + 0.911031 + + kappa + 0.92091 + + kappa + 0.903112 + + kappa + 0.922892 + + kappa + 0.899172 + + kappa + 0.895209 + + kappa + 0.932781 + + kb_relative_information_score + 520.530219 + + kb_relative_information_score + 522.44632 + + kb_relative_information_score + 516.098237 + + kb_relative_information_score + 516.223961 + + kb_relative_information_score + 520.828521 + + kb_relative_information_score + 512.802292 + + kb_relative_information_score + 522.721047 + + kb_relative_information_score + 510.349442 + + kb_relative_information_score + 511.383119 + + kb_relative_information_score + 528.034273 + + mean_absolute_error + 0.01561 + + mean_absolute_error + 0.014946 + + mean_absolute_error + 0.017004 + + mean_absolute_error + 0.01725 + + mean_absolute_error + 0.014984 + + mean_absolute_error + 0.018277 + + mean_absolute_error + 0.014778 + + mean_absolute_error + 0.019306 + + mean_absolute_error + 0.018655 + + mean_absolute_error + 0.01293 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179997 + + mean_prior_absolute_error + 0.179998 + + mean_prior_absolute_error + 0.179998 + + mean_prior_absolute_error + 0.179998 + + mean_prior_absolute_error + 0.179999 + + number_of_instances + 562 [55,57,56,58,57,56,56,56,55,56] + + number_of_instances + 562 [55,57,56,58,57,56,56,56,55,56] + + number_of_instances + 562 [55,57,56,57,57,56,56,57,55,56] + + number_of_instances + 562 [55,57,56,57,57,56,56,57,55,56] + + number_of_instances + 562 [55,57,56,57,57,55,56,57,55,57] + + number_of_instances + 562 [55,57,56,57,57,55,56,57,55,57] + + number_of_instances + 562 [56,57,55,57,57,56,55,57,56,56] + + number_of_instances + 562 [56,57,55,57,57,56,55,57,56,56] + + number_of_instances + 562 [56,58,55,57,56,56,56,56,56,56] + + number_of_instances + 562 [56,57,56,57,56,56,56,56,56,56] + + precision + 0.923823 [1,0.894737,0.981132,0.944444,0.910714,1,0.931034,0.887097,0.87931,0.810345] + + precision + 0.936344 [1,0.980769,0.913793,0.946429,0.913793,0.979592,0.982456,0.963636,0.894737,0.787879] + + precision + 0.922887 [1,0.925926,0.960784,0.912281,0.901639,0.962264,1,0.888889,0.870968,0.807692] + + precision + 0.924699 [1,0.833333,0.979592,1,0.901639,0.98,1,0.933333,0.836066,0.783333] + + precision + 0.92969 [1,0.907407,0.928571,0.95,0.918033,0.942308,0.981481,0.934426,0.942308,0.79661] + + precision + 0.918297 [1,0.870968,0.980392,0.912281,0.945455,0.953488,0.964912,0.965517,0.824561,0.768116] + + precision + 0.934578 [0.981818,0.931034,1,1,0.9,0.910714,0.981481,0.883333,0.944444,0.815385] + + precision + 0.914296 [0.982456,0.9,0.928571,0.945455,0.942308,0.979592,0.964912,0.868852,0.892857,0.73913] + + precision + 0.909699 [1,0.883333,0.942308,0.857143,0.960784,0.979167,1,0.828125,0.872727,0.775862] + + precision + 0.94099 [1,0.915254,0.931034,0.98,0.981818,0.981481,0.982143,0.913793,0.87931,0.844828] + + predictive_accuracy + 0.921708 + + predictive_accuracy + 0.932384 + + predictive_accuracy + 0.921708 + + predictive_accuracy + 0.919929 + + predictive_accuracy + 0.928826 + + predictive_accuracy + 0.912811 + + predictive_accuracy + 0.930605 + + predictive_accuracy + 0.909253 + + predictive_accuracy + 0.905694 + + predictive_accuracy + 0.939502 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + prior_entropy + 3.321833 + + recall + 0.921708 [1,0.894737,0.928571,0.87931,0.894737,0.910714,0.964286,0.982143,0.927273,0.839286] + + recall + 0.932384 [0.981818,0.894737,0.946429,0.913793,0.929825,0.857143,1,0.946429,0.927273,0.928571] + + recall + 0.921708 [0.981818,0.877193,0.875,0.912281,0.964912,0.910714,0.982143,0.982456,0.981818,0.75] + + recall + 0.919929 [0.981818,0.877193,0.857143,0.894737,0.964912,0.875,1,0.982456,0.927273,0.839286] + + recall + 0.928826 [0.963636,0.859649,0.928571,1,0.982456,0.890909,0.946429,1,0.890909,0.824561] + + recall + 0.912811 [0.963636,0.947368,0.892857,0.912281,0.912281,0.745455,0.982143,0.982456,0.854545,0.929825] + + recall + 0.930605 [0.964286,0.947368,0.945455,0.842105,0.947368,0.910714,0.963636,0.929825,0.910714,0.946429] + + recall + 0.909253 [1,0.789474,0.945455,0.912281,0.859649,0.857143,1,0.929825,0.892857,0.910714] + + recall + 0.905694 [0.982143,0.913793,0.890909,0.947368,0.875,0.839286,1,0.946429,0.857143,0.803571] + + recall + 0.939502 [1,0.947368,0.964286,0.859649,0.964286,0.946429,0.982143,0.946429,0.910714,0.875] + + relative_absolute_error + 0.086724 + + relative_absolute_error + 0.083036 + + relative_absolute_error + 0.094469 + + relative_absolute_error + 0.095836 + + relative_absolute_error + 0.083244 + + relative_absolute_error + 0.101539 + + relative_absolute_error + 0.082103 + + relative_absolute_error + 0.107256 + + relative_absolute_error + 0.103639 + + relative_absolute_error + 0.071835 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299997 + + root_mean_prior_squared_error + 0.299998 + + root_mean_prior_squared_error + 0.299998 + + root_mean_prior_squared_error + 0.299999 + + root_mean_prior_squared_error + 0.3 + + root_mean_squared_error + 0.114072 + + root_mean_squared_error + 0.111221 + + root_mean_squared_error + 0.121483 + + root_mean_squared_error + 0.119208 + + root_mean_squared_error + 0.113177 + + root_mean_squared_error + 0.123617 + + root_mean_squared_error + 0.111738 + + root_mean_squared_error + 0.128468 + + root_mean_squared_error + 0.126509 + + root_mean_squared_error + 0.101792 + + root_relative_squared_error + 0.380244 + + root_relative_squared_error + 0.370741 + + root_relative_squared_error + 0.404947 + + root_relative_squared_error + 0.397364 + + root_relative_squared_error + 0.377262 + + root_relative_squared_error + 0.412061 + + root_relative_squared_error + 0.372463 + + root_relative_squared_error + 0.428228 + + root_relative_squared_error + 0.4217 + + root_relative_squared_error + 0.339306 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost + 0 + + total_cost 0 diff --git a/tests/files/setups/1/description.xml b/tests/files/setups/1/description.xml index f918b0573..ee234e4ff 100644 --- a/tests/files/setups/1/description.xml +++ b/tests/files/setups/1/description.xml @@ -1,977 +1,23 @@ - 1 - 56 + 100 + 60 - 228 - 392 - weka.A1DE(2)_F - F - option - 1 - 1 - - - 229 - 392 - weka.A1DE(2)_M - M - option - 1.0 - 1.0 - - - 295 - 376 - weka.AdaBoostM1_DecisionStump(2)_I - I - option - 10 - 10 - - - 296 - 376 - weka.AdaBoostM1_DecisionStump(2)_P - P - option - 100 - 100 - - - 298 - 376 - weka.AdaBoostM1_DecisionStump(2)_S - S - option - 1 - 1 - - - 299 - 376 - weka.AdaBoostM1_DecisionStump(2)_W - W - baselearner - weka.classifiers.trees.DecisionStump - weka.classifiers.trees.DecisionStump - - - 334 - 404 - weka.AdaBoostM1_IBk(2)_I - I - option - 10 - 20 - - - 335 - 404 - weka.AdaBoostM1_IBk(2)_P - P - option - 100 - 100 - - - 337 - 404 - weka.AdaBoostM1_IBk(2)_S - S - option - 1 - 1 - - - 338 - 404 - weka.AdaBoostM1_IBk(2)_W - W - baselearner - weka.classifiers.lazy.IBk - weka.classifiers.lazy.IBk - - - 2883 - 530 - weka.Bagging_NaiveBayes(2)_I - I - option - 10 - 80 - - - 2888 - 530 - weka.Bagging_NaiveBayes(2)_P - P - option - 100 - 100 - - - 2890 - 530 - weka.Bagging_NaiveBayes(2)_S - S - option - 1 - 1 - - - 2892 - 530 - weka.Bagging_NaiveBayes(2)_W - W - baselearner - weka.classifiers.bayes.NaiveBayes - weka.classifiers.bayes.NaiveBayes - - - 2893 - 530 - weka.Bagging_NaiveBayes(2)_num-slots - num-slots - option - 1 - 1 - - - 2909 - 531 - weka.Bagging_OneR(2)_I - I - option - 10 - 160 - - - 2914 - 531 - weka.Bagging_OneR(2)_P - P - option - 100 - 100 - - - 2916 - 531 - weka.Bagging_OneR(2)_S - S - option - 1 - 1 - - - 2918 - 531 - weka.Bagging_OneR(2)_W - W - baselearner - weka.classifiers.rules.OneR - weka.classifiers.rules.OneR - - - 2919 - 531 - weka.Bagging_OneR(2)_num-slots - num-slots - option - 1 - 1 - - - 3132 - 397 - weka.BayesianLogisticRegression(2)_D - D - flag - true - true - - - 3133 - 397 - weka.BayesianLogisticRegression(2)_F - F - option - 2 - 2 - - - 3134 - 397 - weka.BayesianLogisticRegression(2)_H - H - option - 1 - 1 - - - 3135 - 397 - weka.BayesianLogisticRegression(2)_I - I - option - 100 - 100 - - - 3136 - 397 - weka.BayesianLogisticRegression(2)_N - N - flag - true - true - - - 3137 - 397 - weka.BayesianLogisticRegression(2)_P - P - option - 1 - 1 - - - 3138 - 397 - weka.BayesianLogisticRegression(2)_R - R - option - R:0.01-316,3.16 - R:0.01-316,3.16 - - - 3139 - 397 - weka.BayesianLogisticRegression(2)_S - S - option - 0.5 - 0.5 - - - 3140 - 397 - weka.BayesianLogisticRegression(2)_Tl - Tl - option - 5.0E-4 - 5.0E-4 - - - 3141 - 397 - weka.BayesianLogisticRegression(2)_V - V - option - 0.27 - 0.27 - - - 3142 - 397 - weka.BayesianLogisticRegression(2)_seed - seed - option - 1 - 1 - - - 3191 - 441 - weka.ComplementNaiveBayes(2)_S - S - option - 1.0 - 1.0 - - - 3335 - 401 - weka.GaussianProcesses_PolyKernel(3)_L - L - option - 1.0 - 1.0 - - - 3336 - 401 - weka.GaussianProcesses_PolyKernel(3)_N - N - option - 0 - 0 - - - 3409 - 389 - weka.IBk(2)_A - A - flag - true - true - - - 3413 - 389 - weka.IBk(2)_K - K - option - 1 - 1 - - - 3414 - 389 - weka.IBk(2)_W - W - option - 0 - 0 - - - 3951 - 375 - weka.OneR(3)_B - B - option - 6 - 6 - - - 4028 - 386 - weka.PolyKernel(4)_C - C - option - 250007 - 250007 - - - 4029 - 386 - weka.PolyKernel(4)_E - E - option - 1.0 - 1.0 - - - 5431 - 564 - weka.GaussianProcesses_NormalizedPolyKernel(2)_L - L - option - 1.0 - 1.0 - - - 5432 - 564 - weka.GaussianProcesses_NormalizedPolyKernel(2)_N - N - option - 0 - 0 - - - 5433 - 564 - weka.GaussianProcesses_NormalizedPolyKernel(2)_K - K - kernel - weka.classifiers.functions.supportVector.NormalizedPolyKernel - weka.classifiers.functions.supportVector.NormalizedPolyKernel - - - 5439 - 565 - weka.NormalizedPolyKernel(2)_E - E - option - 2.0 - 2.0 - - - 5441 - 565 - weka.NormalizedPolyKernel(2)_C - C - option - 250007 - 250007 - - - 5566 - 582 - weka.CVParameterSelection_ZeroR(2)_X - X - option - 10 - 10 - - - 5568 - 582 - weka.CVParameterSelection_ZeroR(2)_S - S - option - 1 - 1 - - - 5569 - 582 - weka.CVParameterSelection_ZeroR(2)_W - W - baselearner - weka.classifiers.rules.ZeroR - weka.classifiers.rules.ZeroR - - - 6514 - 675 - weka.J48(13)_C - C - option - 0.25 - 0.25 - - - 6515 - 675 - weka.J48(13)_M - M - option - 2 - 2 - - - 9551 - 1068 - weka.J48(28)_C + 3432 + 60 + weka.J48(1)_C C option 0.25 - 0.25 - - - 9552 - 1068 - weka.J48(28)_M - M - option - 2 - 2 + 0.9 - 9601 - 1077 - weka.REPTree(9)_M + 3435 + 60 + weka.J48(1)_M M option 2 2 - - 9602 - 1077 - weka.REPTree(9)_V - V - option - 0.001 - 0.001 - - - 9603 - 1077 - weka.REPTree(9)_N - N - option - 3 - 3 - - - 9604 - 1077 - weka.REPTree(9)_S - S - option - 1 - 1 - - - 9606 - 1077 - weka.REPTree(9)_L - L - option - -1 - -1 - - - 9607 - 1077 - weka.REPTree(9)_I - I - option - 0.0 - 0.0 - - - 9611 - 1078 - weka.RandomTree(10)_K - K - option - 0 - 0 - - - 9612 - 1078 - weka.RandomTree(10)_M - M - option - 1.0 - 1.0 - - - 9613 - 1078 - weka.RandomTree(10)_V - V - option - 0.001 - 0.001 - - - 9614 - 1078 - weka.RandomTree(10)_S - S - option - 1 - 1 - - - 9620 - 1079 - weka.RandomForest(5)_I - I - option - 100 - 100 - - - 9621 - 1079 - weka.RandomForest(5)_K - K - option - 0 - 0 - - - 9622 - 1079 - weka.RandomForest(5)_S - S - option - 1 - 1 - - - 9626 - 1079 - weka.RandomForest(5)_num-slots - num-slots - option - 1 - 1 - - - 9873 - 1116 - weka.MultilayerPerceptronCS(1)_H - H - option - a - a - - - 10227 - 1165 - weka.A1DE(4)_F - F - option - 1 - 1 - - - 10228 - 1165 - weka.A1DE(4)_M - M - option - 1.0 - 1.0 - - - 10246 - 1168 - weka.BayesNet_K2(6)_D - D - flag - true - true - - - 10248 - 1168 - weka.BayesNet_K2(6)_Q - Q - baselearner - weka.classifiers.bayes.net.search.local.K2 - weka.classifiers.bayes.net.search.local.K2 - - - 10259 - 1169 - weka.K2(5)_P - P - option - 1 - 1 - - - 10262 - 1169 - weka.K2(5)_S - S - option - BAYES - BAYES - - - 10268 - 1172 - weka.LibSVM(2)_S - S - option - 0 - 0 - - - 10269 - 1172 - weka.LibSVM(2)_K - K - option - 2 - 2 - - - 10270 - 1172 - weka.LibSVM(2)_D - D - option - 3 - 3 - - - 10271 - 1172 - weka.LibSVM(2)_G - G - option - 0.0 - 0.0 - - - 10272 - 1172 - weka.LibSVM(2)_R - R - option - 0.0 - 0.0 - - - 10273 - 1172 - weka.LibSVM(2)_C - C - option - 1.0 - 1.0 - - - 10274 - 1172 - weka.LibSVM(2)_N - N - option - 0.5 - 0.5 - - - 10278 - 1172 - weka.LibSVM(2)_P - P - option - 0.1 - 0.1 - - - 10279 - 1172 - weka.LibSVM(2)_M - M - option - 40.0 - 40.0 - - - 10280 - 1172 - weka.LibSVM(2)_E - E - option - 0.001 - 0.001 - - - 10284 - 1172 - weka.LibSVM(2)_model - model - option - /Users/joa/Downloads/weka-3-7-12 - /Users/joa/Downloads/weka-3-7-12 - - - 10285 - 1172 - weka.LibSVM(2)_seed - seed - option - 1 - 1 - - - 10290 - 1174 - weka.KernelLogisticRegression_RBFKernel(1)_S - S - option - 1 - 1 - - - 10293 - 1174 - weka.KernelLogisticRegression_RBFKernel(1)_K - K - kernel - weka.classifiers.functions.supportVector.RBFKernel - weka.classifiers.functions.supportVector.RBFKernel - - - 10294 - 1174 - weka.KernelLogisticRegression_RBFKernel(1)_L - L - option - 0.01 - 0.01 - - - 10296 - 1174 - weka.KernelLogisticRegression_RBFKernel(1)_P - P - option - 1 - 1 - - - 10297 - 1174 - weka.KernelLogisticRegression_RBFKernel(1)_E - E - option - 1 - 1 - - - 10300 - 1175 - weka.RBFKernel(3)_G - G - option - 0.01 - 0.01 - - - 10301 - 1175 - weka.RBFKernel(3)_C - C - option - 250007 - 250007 - - - 10385 - 1185 - weka.Bagging_REPTree(8)_P - P - option - 100 - 100 - - - 10388 - 1185 - weka.Bagging_REPTree(8)_S - S - option - 1 - 1 - - - 10389 - 1185 - weka.Bagging_REPTree(8)_num-slots - num-slots - option - 1 - 1 - - - 10390 - 1185 - weka.Bagging_REPTree(8)_I - I - option - 10 - 10 - - - 10391 - 1185 - weka.Bagging_REPTree(8)_W - W - baselearner - weka.classifiers.trees.REPTree - weka.classifiers.trees.REPTree - - - 10547 - 1199 - weka.ADTree(4)_B - B - option - 10 - 10 - - - 10548 - 1199 - weka.ADTree(4)_E - E - option - -3 - -3 - - - 10550 - 1199 - weka.ADTree(4)_S - S - option - 1 - 1 - - - 10553 - 1200 - weka.LADTree(4)_B - B - option - 10 - 10 - - - 10749 - 1244 - weka.FilteredClassifier_Discretize_J48(4)_F - F - kernel - weka.filters.supervised.attribute.Discretize - weka.filters.supervised.attribute.Discretize - - - 10750 - 1244 - weka.FilteredClassifier_Discretize_J48(4)_W - W - baselearner - weka.classifiers.trees.J48 - weka.classifiers.trees.J48 - - - 10772 - 1245 - weka.Discretize(3)_R - R - option - first-last - first-last - - - 10778 - 1245 - weka.Discretize(3)_precision - precision - option - 6 - 6 - - - 14254 - 1717 - weka.RandomForest(5)_K - K - option - 0 - 0 - - - 14256 - 1717 - weka.RandomForest(5)_S - S - option - 1 - 1 - - - 14258 - 1717 - weka.RandomForest(5)_num-slots - num-slots - option - 1 - 1 - From 6a60427ef816ef46c1263bc272df428ea7ef25f7 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Sun, 11 Feb 2018 12:39:15 +0100 Subject: [PATCH 60/86] changed cache directory creation (for python 2) --- openml/runs/functions.py | 4 +--- openml/setups/functions.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 66ab879e6..6a4292b36 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -641,10 +641,8 @@ def get_run(run_id): run_dir = os.path.join(config.get_cache_directory(), "runs", str(run_id)) run_file = os.path.join(run_dir, "description.xml") - try: + if not os.path.exists(run_dir): os.makedirs(run_dir) - except FileExistsError: - pass try: return _get_cached_run(run_id) diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 492129af2..c0e256607 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -90,10 +90,8 @@ def get_setup(setup_id): setup_dir = os.path.join(config.get_cache_directory(), "setups", str(setup_id)) setup_file = os.path.join(setup_dir, "description.xml") - try: + if not os.path.exists(setup_dir): os.makedirs(setup_dir) - except FileExistsError: - pass try: return _get_cached_setup(setup_id) From 8427b1a006bd3ae0b70619f2e55134d6e6fe4c9d Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Mon, 12 Feb 2018 11:07:44 +0100 Subject: [PATCH 61/86] skipped additional doc test --- doc/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/usage.rst b/doc/usage.rst index 61a223af4..0801c2c03 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -143,7 +143,7 @@ We can filter the list of tasks to only contain datasets with more than >>> filtered_tasks = tasks.query('NumberOfInstances > 500 and NumberOfInstances < 1000') >>> print(list(filtered_tasks.index)) # doctest: +SKIP [2, 11, 15, 29, 37, 41, 49, 53, ..., 146597, 146600, 146605] - >>> print(len(filtered_tasks)) + >>> print(len(filtered_tasks)) # doctest: +SKIP 210 Then, we can further restrict the tasks to all have the same resampling From 34a86a1773b7de1e3f57b18d02e8e1d74dc62eb3 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Fri, 16 Feb 2018 13:36:24 +0100 Subject: [PATCH 62/86] incorporated changes requested by #mfeurer --- openml/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openml/utils.py b/openml/utils.py index 18dcd9fbf..c80fa0593 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -45,8 +45,10 @@ def extract_xml_tags(xml_tag_name, node, allow_none=True): def _tag_entity(entity_type, entity_id, tag, untag=False): - """Abstract function that can be used as a partial for tagging entities - on OpenML + """Function that tags or untags a given entity on OpenML. As the OpenML + API tag functions all consist of the same format, this function covers + all entity types (currently: dataset, task, flow, setup, run). Could + be used in a partial to provide dataset_tag, dataset_untag, etc. Parameters ---------- @@ -65,7 +67,7 @@ def _tag_entity(entity_type, entity_id, tag, untag=False): Returns ------- tags : list - List of tags that the entity is still tagged with + List of tags that the entity is (still) tagged with """ legal_entities = {'data', 'task', 'flow', 'setup', 'run'} if entity_type not in legal_entities: From db65bdc7d9b67f1ff746eed063953659776527ed Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Mon, 19 Feb 2018 10:16:45 +0100 Subject: [PATCH 63/86] added list of integers to set of accepted parameter types for arff traces --- openml/runs/functions.py | 15 ++++++++++----- tests/test_runs/test_run_functions.py | 6 ++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 5a3b4bee1..32693865b 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -594,11 +594,16 @@ def _extract_arfftrace_attributes(model): for key in model.cv_results_: if key.startswith('param_'): # supported types should include all types, including bool, int float - supported_types = (bool, int, float, six.string_types, tuple) - if all(isinstance(i, supported_types) or i is None for i in model.cv_results_[key]): - type = 'STRING' - else: - raise TypeError('Unsupported param type in param grid') + supported_basic_types = (bool, int, float, six.string_types) + for param_value in model.cv_results_[key]: + if isinstance(param_value, supported_basic_types) or param_value is None: + # basic string values + type = 'STRING' + elif isinstance(param_value, list) and all(isinstance(i, int) for i in param_value): + # list of integers + type = 'STRING' + else: + raise TypeError('Unsupported param type in param grid: %s' %key) # we renamed the attribute param to parameter, as this is a required # OpenML convention diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index ccce63378..5d38ab8e4 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -615,7 +615,7 @@ def test__get_seeded_model_raises(self): self.assertRaises(ValueError, _get_seeded_model, model=clf, seed=42) def test__extract_arfftrace(self): - param_grid = {"hidden_layer_sizes": [(5, 5), (10, 10), (20, 20)], + param_grid = {"hidden_layer_sizes": [[5, 5], [10, 10], [20, 20]], "activation" : ['identity', 'logistic', 'tanh', 'relu'], "learning_rate_init": [0.1, 0.01, 0.001, 0.0001], "max_iter": [10, 20, 40, 80]} @@ -627,6 +627,9 @@ def test__extract_arfftrace(self): X, y = task.get_X_and_y() clf.fit(X[train], y[train]) + # check num layers of MLP + self.assertIn(clf.best_estimator_.hidden_layer_sizes, param_grid['hidden_layer_sizes']) + trace_attribute_list = _extract_arfftrace_attributes(clf) trace_list = _extract_arfftrace(clf, 0, 0) self.assertIsInstance(trace_attribute_list, list) @@ -660,7 +663,6 @@ def test__extract_arfftrace(self): else: # att_type = real self.assertIsInstance(trace_list[line_idx][att_idx], float) - self.assertEqual(set(param_grid.keys()), optimized_params) def test__prediction_to_row(self): From e4035c193e9ceb287726399d7eb4bb20631d5fba Mon Sep 17 00:00:00 2001 From: Joaquin Vanschoren Date: Tue, 20 Feb 2018 08:48:15 +0100 Subject: [PATCH 64/86] fix for list_all. It currently keeps looping forever (until the connection breaks) (#390) --- openml/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openml/utils.py b/openml/utils.py index c80fa0593..cc976b4c3 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -118,9 +118,10 @@ def list_all(listing_call, batch_size=10000, *args, **filters): dict """ page = 0 + has_more = 1 result = {} - while True: + while has_more: try: new_batch = listing_call( *args, @@ -135,5 +136,6 @@ def list_all(listing_call, batch_size=10000, *args, **filters): break result.update(new_batch) page += 1 + has_more = (len(new_batch) == batch_size) return result From 4cfe59a865dc575e055a4cb9f2a523a86b94a6de Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 20 Feb 2018 17:57:51 +0100 Subject: [PATCH 65/86] added status option to datalist --- openml/datasets/functions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index f6dea2cfb..a586090d4 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -137,7 +137,7 @@ def _get_cached_dataset_arff(dataset_id): "cached" % dataset_id) -def list_datasets(offset=None, size=None, tag=None): +def list_datasets(offset=None, size=None, tag=None, status=None): """Return a list of all dataset which are on OpenML. Parameters @@ -148,6 +148,10 @@ def list_datasets(offset=None, size=None, tag=None): The maximum number of datasets to show. tag : str, optional Only include datasets matching this tag. + status : str, optional + Should be {active, in_preparation, deactivated}. By + default active datasets are returned, but also datasets + from another status can be requested. Returns ------- @@ -174,6 +178,9 @@ def list_datasets(offset=None, size=None, tag=None): if tag is not None: api_call += "/tag/%s" % tag + if status is not None: + api_call += "/status/%s" %status + return _list_datasets(api_call) From a1e9368b3aba291c1ee670977a8bde5a091d7ef9 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Thu, 22 Feb 2018 18:01:47 +0100 Subject: [PATCH 66/86] fixes empty listing bug (#405) --- openml/_api_calls.py | 6 +++--- openml/datasets/functions.py | 2 +- openml/evaluations/functions.py | 7 +++++-- openml/flows/functions.py | 2 +- openml/runs/functions.py | 2 +- openml/setups/functions.py | 6 +++++- openml/study/functions.py | 1 + openml/tasks/functions.py | 2 +- tests/test_datasets/test_dataset_functions.py | 8 ++++++++ tests/test_evaluations/test_evaluation_functions.py | 7 +++++++ tests/test_flows/test_flow_functions.py | 7 +++++++ tests/test_runs/test_run_functions.py | 7 +++++++ tests/test_setups/test_setup_functions.py | 7 +++++++ tests/test_tasks/test_task_functions.py | 7 +++++++ 14 files changed, 61 insertions(+), 10 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 81a3d7756..93f0ed2f1 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -139,9 +139,9 @@ def _parse_server_exception(response, url=None): additional = None if 'oml:additional_information' in server_exception['oml:error']: additional = server_exception['oml:error']['oml:additional_information'] - if code in [370, 372, 512, 500, 482]: - # 512 for runs, 370 for datasets (should be 372), 500 for flows - # 482 for tasks + if code in [372, 512, 500, 482, 542, 674]: # datasets, + # 512 for runs, 372 for datasets, 500 for flows + # 482 for tasks, 542 for evaluations, 674 for setups return OpenMLServerNoResult(code, message, additional) return OpenMLServerException( code=code, diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index a586090d4..b9a1079be 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -189,7 +189,7 @@ def _list_datasets(api_call): try: xml_string = _perform_api_call(api_call) except OpenMLServerNoResult: - return [] + return dict() datasets_dict = xmltodict.parse(xml_string, force_list=('oml:dataset',)) # Minimalistic check if the XML is useful diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index aa7f86635..c3e0d9914 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -1,5 +1,6 @@ import xmltodict +from openml.exceptions import OpenMLServerNoResult from .._api_calls import _perform_api_call from ..evaluations import OpenMLEvaluation @@ -59,8 +60,10 @@ def list_evaluations(function, offset=None, size=None, id=None, task=None, def _list_evaluations(api_call): """Helper function to parse API calls which are lists of runs""" - - xml_string = _perform_api_call(api_call) + try: + xml_string = _perform_api_call(api_call) + except OpenMLServerNoResult: + return dict() evals_dict = xmltodict.parse(xml_string, force_list=('oml:evaluation',)) # Minimalistic check if the XML is useful diff --git a/openml/flows/functions.py b/openml/flows/functions.py index bd8467a42..61a260f35 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -112,7 +112,7 @@ def _list_flows(api_call): try: xml_string = _perform_api_call(api_call) except OpenMLServerNoResult: - return [] + return dict() flows_dict = xmltodict.parse(xml_string, force_list=('oml:flow',)) # Minimalistic check if the XML is useful diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 6a4292b36..8a28fa528 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -948,7 +948,7 @@ def _list_runs(api_call): try: xml_string = _perform_api_call(api_call) except OpenMLServerNoResult: - return [] + return dict() runs_dict = xmltodict.parse(xml_string, force_list=('oml:run',)) # Minimalistic check if the XML is useful diff --git a/openml/setups/functions.py b/openml/setups/functions.py index c0e256607..a78e07ae6 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -8,6 +8,7 @@ from .. import config from .setup import OpenMLSetup, OpenMLParameter from openml.flows import flow_exists +from openml.exceptions import OpenMLServerNoResult def setup_exists(flow, model=None): @@ -145,7 +146,10 @@ def list_setups(flow=None, tag=None, setup=None, offset=None, size=None): def _list_setups(api_call): """Helper function to parse API calls which are lists of setups""" - xml_string = openml._api_calls._perform_api_call(api_call) + try: + xml_string = openml._api_calls._perform_api_call(api_call) + except OpenMLServerNoResult: + return dict() setups_dict = xmltodict.parse(xml_string, force_list=('oml:setup',)) # Minimalistic check if the XML is useful diff --git a/openml/study/functions.py b/openml/study/functions.py index 11c47d674..535cf8dcd 100644 --- a/openml/study/functions.py +++ b/openml/study/functions.py @@ -3,6 +3,7 @@ from openml.study import OpenMLStudy from .._api_calls import _perform_api_call + def _multitag_to_list(result_dict, tag): if isinstance(result_dict[tag], list): return result_dict[tag] diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 31a76eb48..32c4c0fec 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -141,7 +141,7 @@ def _list_tasks(api_call): try: xml_string = _perform_api_call(api_call) except OpenMLServerNoResult: - return [] + return dict() tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task','oml:input')) # Minimalistic check if the XML is useful if 'oml:tasks' not in tasks_dict: diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 35db14ec0..0f55b503d 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -147,6 +147,14 @@ def test_list_datasets_paginate(self): for did in datasets: self._check_dataset(datasets[did]) + def test_list_datasets_empty(self): + datasets = openml.datasets.list_datasets(tag='NoOneWouldUseThisTagAnyway') + if len(datasets) > 0: + raise ValueError('UnitTest Outdated, tag was already used (please remove)') + + self.assertIsInstance(datasets, dict) + + @unittest.skip('See https://github.com/openml/openml-python/issues/149') def test_check_datasets_active(self): active = openml.datasets.check_datasets_active([1, 17]) diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 47e6d72e4..771ee2cd4 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -63,3 +63,10 @@ def test_evaluation_list_limit(self): evaluations = openml.evaluations.list_evaluations("predictive_accuracy", size=100, offset=100) self.assertEquals(len(evaluations), 100) + + def test_list_evaluations_empty(self): + evaluations = openml.evaluations.list_evaluations('unexisting_measure') + if len(evaluations) > 0: + raise ValueError('UnitTest Outdated, got somehow results') + + self.assertIsInstance(evaluations, dict) \ No newline at end of file diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 47e04581b..419b86f13 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -31,6 +31,13 @@ def test_list_flows(self): for fid in flows: self._check_flow(flows[fid]) + def test_list_flows_empty(self): + flows = openml.flows.list_flows(tag='NoOneEverUsesThisTag123') + if len(flows) > 0: + raise ValueError('UnitTest Outdated, got somehow results (please adapt)') + + self.assertIsInstance(flows, dict) + def test_list_flows_by_tag(self): flows = openml.flows.list_flows(tag='weka') self.assertGreaterEqual(len(flows), 5) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index e2fd5b286..56abdd266 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -830,6 +830,13 @@ def test_get_runs_list(self): for rid in runs: self._check_run(runs[rid]) + def test_list_runs_empty(self): + runs = openml.runs.list_runs(task=[-1]) + if len(runs) > 0: + raise ValueError('UnitTest Outdated, got somehow results') + + self.assertIsInstance(runs, dict) + def test_get_runs_list_by_task(self): # TODO: comes from live, no such lists on test openml.config.server = self.production_server diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 85cb8419f..9dffe5a04 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -137,6 +137,13 @@ def test_setup_list_filter_flow(self): for setup_id in setups.keys(): self.assertEquals(setups[setup_id].flow_id, flow_id) + def test_list_setups_empty(self): + setups = openml.setups.list_setups(setup=[-1]) + if len(setups) > 0: + raise ValueError('UnitTest Outdated, got somehow results') + + self.assertIsInstance(setups, dict) + def test_setuplist_offset(self): # TODO: remove after pull on live for better testing # openml.config.server = self.production_server diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index ea84a27c7..21cc9c0e2 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -67,6 +67,13 @@ def test_list_tasks_by_type(self): self.assertEquals(ttid, tasks[tid]["ttid"]) self._check_task(tasks[tid]) + def test_list_tasks_empty(self): + tasks = openml.tasks.list_tasks(tag='NoOneWillEverUseThisTag') + if len(tasks) > 0: + raise ValueError('UnitTest Outdated, got somehow results (tag is used, please adapt)') + + self.assertIsInstance(tasks, dict) + def test_list_tasks_by_tag(self): num_basic_tasks = 100 # number is flexible, check server if fails tasks = openml.tasks.list_tasks(tag='study_14') From 02560d7bf887cbf5e0e34832945e5b40c78d5e43 Mon Sep 17 00:00:00 2001 From: "janvanrijn@gmail.com" Date: Tue, 13 Mar 2018 12:25:42 -0400 Subject: [PATCH 67/86] fix unit test --- tests/test_setups/test_setup_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 9dffe5a04..e2c705a6e 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -138,7 +138,7 @@ def test_setup_list_filter_flow(self): self.assertEquals(setups[setup_id].flow_id, flow_id) def test_list_setups_empty(self): - setups = openml.setups.list_setups(setup=[-1]) + setups = openml.setups.list_setups(setup=[0]) if len(setups) > 0: raise ValueError('UnitTest Outdated, got somehow results') From deb769e500286457d6b12483fc47403912cd1dce Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Wed, 14 Mar 2018 17:21:38 +0100 Subject: [PATCH 68/86] Fix #378 (#418) * Fix ascii decoding problem with python 2 * Created separate pickle files for splits and datasets according to the python version * Added production task to unit test * Update test_task_functions.py --- openml/datasets/data_feature.py | 7 ++++++- openml/datasets/dataset.py | 5 ++++- openml/tasks/split.py | 15 +++++++++++---- tests/test_tasks/test_task_functions.py | 4 ++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/openml/datasets/data_feature.py b/openml/datasets/data_feature.py index 627d92745..51b132f1c 100644 --- a/openml/datasets/data_feature.py +++ b/openml/datasets/data_feature.py @@ -1,3 +1,4 @@ +import six class OpenMLDataFeature(object): """Data Feature (a.k.a. Attribute) object. @@ -29,7 +30,11 @@ def __init__(self, index, name, data_type, nominal_values, raise ValueError('number_missing_values is of wrong datatype') self.index = index - self.name = str(name) + # In case of python version lower than 3, change the default ASCII encoder. + if six.PY2: + self.name = str(name.encode('utf8')) + else: + self.name = str(name) self.data_type = str(data_type) self.nominal_values = nominal_values self.number_missing_values = number_missing_values diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 85ef0cbcb..8761837eb 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -91,7 +91,10 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None, if data_file is not None: if self._data_features_supported(): - self.data_pickle_file = data_file.replace('.arff', '.pkl') + if six.PY2: + self.data_pickle_file = data_file.replace('.arff', '.pkl.py2') + else: + self.data_pickle_file = data_file.replace('.arff', '.pkl.py3') if os.path.exists(self.data_pickle_file): logger.debug("Data pickle file already exists.") diff --git a/openml/tasks/split.py b/openml/tasks/split.py index 6b7c7d0eb..ae7f3a85f 100644 --- a/openml/tasks/split.py +++ b/openml/tasks/split.py @@ -1,6 +1,6 @@ from collections import namedtuple, OrderedDict import os -import sys +import six import numpy as np import scipy.io.arff @@ -60,11 +60,18 @@ def __eq__(self, other): @classmethod def _from_arff_file(cls, filename, cache=True): repetitions = None - pkl_filename = filename.replace(".arff", ".pkl") + if six.PY2: + pkl_filename = filename.replace(".arff", ".pkl.py2") + else: + pkl_filename = filename.replace(".arff", ".pkl.py3") if cache: if os.path.exists(pkl_filename): - with open(pkl_filename, "rb") as fh: - _ = pickle.load(fh) + try: + with open(pkl_filename, "rb") as fh: + _ = pickle.load(fh) + except UnicodeDecodeError as e: + # Possibly pickle file was created with python2 and python3 is being used to load the data + raise e repetitions = _["repetitions"] name = _["name"] diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 21cc9c0e2..b9d4368e7 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -111,6 +111,10 @@ def test_list_tasks_per_type_paginate(self): def test__get_task(self): openml.config.set_cache_directory(self.static_cache_dir) task = openml.tasks.get_task(1882) + # Test the following task as it used to throw an Unicode Error. + # https://github.com/openml/openml-python/issues/378 + openml.config.server = self.production_server + production_task = openml.tasks.get_task(34536) def test_get_task(self): task = openml.tasks.get_task(1) From 3e99d99527ab5ad9294567505535a41acf8e2d37 Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Mon, 19 Mar 2018 14:32:32 +0100 Subject: [PATCH 69/86] Feature #369 (#424) * Implementing dataset listing with more filters and adding unit tests * Add to function documentation --- openml/datasets/functions.py | 16 ++++---- tests/test_datasets/test_dataset_functions.py | 37 ++++++++++++++++--- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index b9a1079be..f2212145d 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -137,7 +137,7 @@ def _get_cached_dataset_arff(dataset_id): "cached" % dataset_id) -def list_datasets(offset=None, size=None, tag=None, status=None): +def list_datasets(offset=None, size=None, status=None, **kwargs): """Return a list of all dataset which are on OpenML. Parameters @@ -146,12 +146,13 @@ def list_datasets(offset=None, size=None, tag=None, status=None): The number of datasets to skip, starting from the first. size : int, optional The maximum number of datasets to show. - tag : str, optional - Only include datasets matching this tag. status : str, optional Should be {active, in_preparation, deactivated}. By default active datasets are returned, but also datasets - from another status can be requested. + from another status can be requested. + kwargs : dict, optional + Legal filter operators (keys in the dict): + {tag, status, limit, offset, data_name, data_version, number_instances, number_features, number_classes, number_missing_values}. Returns ------- @@ -175,12 +176,13 @@ def list_datasets(offset=None, size=None, tag=None, status=None): if size is not None: api_call += "/limit/%d" % int(size) - if tag is not None: - api_call += "/tag/%s" % tag - if status is not None: api_call += "/status/%s" %status + if kwargs is not None: + for filter, value in kwargs.items(): + api_call += "/%s/%s" % (filter, value) + return _list_datasets(api_call) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 0f55b503d..85986fdf1 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -115,6 +115,9 @@ def _check_dataset(self, dataset): self.assertIsInstance(dataset['status'], six.string_types) self.assertIn(dataset['status'], ['in_preparation', 'active', 'deactivated']) + def _check_datasets(self, datasets): + for did in datasets: + self._check_dataset(datasets[did]) def test_tag_untag_dataset(self): tag = 'test_tag_%d' %random.randint(1, 1000000) @@ -129,14 +132,37 @@ def test_list_datasets(self): datasets = openml.datasets.list_datasets() # 1087 as the number of datasets on openml.org self.assertGreaterEqual(len(datasets), 100) - for did in datasets: - self._check_dataset(datasets[did]) + self._check_datasets(datasets) def test_list_datasets_by_tag(self): datasets = openml.datasets.list_datasets(tag='study_14') self.assertGreaterEqual(len(datasets), 100) - for did in datasets: - self._check_dataset(datasets[did]) + self._check_datasets(datasets) + + def test_list_datasets_by_number_instances(self): + datasets = openml.datasets.list_datasets(number_instances="5..100") + self.assertGreaterEqual(len(datasets), 4) + self._check_datasets(datasets) + + def test_list_datasets_by_number_features(self): + datasets = openml.datasets.list_datasets(number_features="50..100") + self.assertGreaterEqual(len(datasets), 8) + self._check_datasets(datasets) + + def test_list_datasets_by_number_classes(self): + datasets = openml.datasets.list_datasets(number_classes="5") + self.assertGreaterEqual(len(datasets), 3) + self._check_datasets(datasets) + + def test_list_datasets_by_number_missing_values(self): + datasets = openml.datasets.list_datasets(number_missing_values="5..100") + self.assertGreaterEqual(len(datasets), 5) + self._check_datasets(datasets) + + def test_list_datasets_combined_filters(self): + datasets = openml.datasets.list_datasets(tag='study_14', number_instances="100..1000", number_missing_values="800..1000") + self.assertGreaterEqual(len(datasets), 1) + self._check_datasets(datasets) def test_list_datasets_paginate(self): size = 10 @@ -144,8 +170,7 @@ def test_list_datasets_paginate(self): for i in range(0, max, size): datasets = openml.datasets.list_datasets(offset=i, size=size) self.assertGreaterEqual(size, len(datasets)) - for did in datasets: - self._check_dataset(datasets[did]) + self._check_datasets(datasets) def test_list_datasets_empty(self): datasets = openml.datasets.list_datasets(tag='NoOneWouldUseThisTagAnyway') From 25136676c51cb6ad64d104ae77fa91067cdf60a5 Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Wed, 28 Mar 2018 18:03:47 +0200 Subject: [PATCH 70/86] Paging (#426) * Created first basic template, removed redudant variable * Improving list_datasets * First implementation of the list_* with the limit tag active * First implementation of the feature, fixed bugs and refactored the code * Changing batch_size to be a keyword argument * Fixed not considering initial offset, removing size and the double offset key from the filter dict * Changing task_type_id argument name in accordance with the new implementation * Reverting previous solution for task_type_id, implementing another fix * Fix for python2 and changing the unit test which times out * Added another test method and did a slight change in an existing test method * Changing the assert value for the failing test method * Added the implementation to filter by uploader for flows, filter by task_type for runs, filter by multipple operator for tasks and also refactored the code according to PEP8 * Refactored code as requested --- openml/datasets/functions.py | 56 +++++++++----- openml/evaluations/functions.py | 68 ++++++++++++----- openml/flows/functions.py | 52 ++++++++----- openml/runs/functions.py | 76 ++++++++++++++----- openml/setups/functions.py | 57 ++++++++------ openml/tasks/functions.py | 74 +++++++++++++----- openml/utils.py | 48 ++++++++---- tests/test_datasets/test_dataset_functions.py | 7 +- tests/test_runs/test_run_functions.py | 6 +- 9 files changed, 306 insertions(+), 138 deletions(-) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index f2212145d..6e3123bce 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -8,6 +8,7 @@ from oslo_concurrency import lockutils import xmltodict +import openml.utils from .dataset import OpenMLDataset from ..exceptions import OpenMLCacheException, OpenMLServerNoResult from .. import config @@ -137,8 +138,10 @@ def _get_cached_dataset_arff(dataset_id): "cached" % dataset_id) -def list_datasets(offset=None, size=None, status=None, **kwargs): - """Return a list of all dataset which are on OpenML. +def list_datasets(offset=None, size=None, status=None, tag=None, **kwargs): + + """ + Return a list of all dataset which are on OpenML. (Supports large amount of results) Parameters ---------- @@ -150,9 +153,11 @@ def list_datasets(offset=None, size=None, status=None, **kwargs): Should be {active, in_preparation, deactivated}. By default active datasets are returned, but also datasets from another status can be requested. + tag : str, optional kwargs : dict, optional Legal filter operators (keys in the dict): - {tag, status, limit, offset, data_name, data_version, number_instances, number_features, number_classes, number_missing_values}. + data_name, data_version, number_instances, + number_features, number_classes, number_missing_values. Returns ------- @@ -169,29 +174,38 @@ def list_datasets(offset=None, size=None, status=None, **kwargs): If qualities are calculated for the dataset, some of these are also returned. """ - api_call = "data/list" - if offset is not None: - api_call += "/offset/%d" % int(offset) - if size is not None: - api_call += "/limit/%d" % int(size) + return openml.utils.list_all(_list_datasets, offset=offset, size=size, status=status, tag=tag, **kwargs) - if status is not None: - api_call += "/status/%s" %status + +def _list_datasets(**kwargs): + + """ + Perform api call to return a list of all datasets. + + Parameters + ---------- + kwargs : dict, optional + Legal filter operators (keys in the dict): + {tag, status, limit, offset, data_name, data_version, number_instances, + number_features, number_classes, number_missing_values. + + Returns + ------- + datasets : dict of dicts + """ + + api_call = "data/list" if kwargs is not None: - for filter, value in kwargs.items(): - api_call += "/%s/%s" % (filter, value) + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) + return __list_datasets(api_call) - return _list_datasets(api_call) +def __list_datasets(api_call): -def _list_datasets(api_call): - # TODO add proper error handling here! - try: - xml_string = _perform_api_call(api_call) - except OpenMLServerNoResult: - return dict() + xml_string = _perform_api_call(api_call) datasets_dict = xmltodict.parse(xml_string, force_list=('oml:dataset',)) # Minimalistic check if the XML is useful @@ -224,7 +238,7 @@ def check_datasets_active(dataset_ids): Parameters ---------- - dataset_id : iterable + dataset_ids : iterable Integers representing dataset ids. Returns @@ -279,7 +293,7 @@ def get_dataset(dataset_id): Parameters ---------- - ddataset_id : int + dataset_id : int Dataset ID of the dataset to download Returns diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index c3e0d9914..9711fd574 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -1,19 +1,20 @@ import xmltodict from openml.exceptions import OpenMLServerNoResult +import openml.utils from .._api_calls import _perform_api_call from ..evaluations import OpenMLEvaluation def list_evaluations(function, offset=None, size=None, id=None, task=None, setup=None, flow=None, uploader=None, tag=None): - """List all run-evaluation pairs matching all of the given filters. + """ + List all run-evaluation pairs matching all of the given filters. + (Supports large amount of results) - Perform API call ``/evaluation/function{function}/{filters}`` - Parameters ---------- - function : str + function : str the evaluation function. e.g., predictive_accuracy offset : int, optional the number of runs to skip, starting from the first @@ -37,11 +38,45 @@ def list_evaluations(function, offset=None, size=None, id=None, task=None, dict """ - api_call = "evaluation/list/function/%s" %function - if offset is not None: - api_call += "/offset/%d" % int(offset) - if size is not None: - api_call += "/limit/%d" % int(size) + return openml.utils.list_all(_list_evaluations, function, offset=offset, size=size, + id=id, task=task, setup=setup, flow=flow, uploader=uploader, tag=tag) + + +def _list_evaluations(function, id=None, task=None, + setup=None, flow=None, uploader=None, **kwargs): + """ + Perform API call ``/evaluation/function{function}/{filters}`` + + Parameters + ---------- + The arguments that are lists are separated from the single value + ones which are put into the kwargs. + + function : str + the evaluation function. e.g., predictive_accuracy + + id : list, optional + + task : list, optional + + setup: list, optional + + flow : list, optional + + uploader : list, optional + + kwargs: dict, optional + Legal filter operators: tag, limit, offset. + + Returns + ------- + dict + """ + + api_call = "evaluation/list/function/%s" % function + if kwargs is not None: + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) if id is not None: api_call += "/run/%s" % ','.join([str(int(i)) for i in id]) if task is not None: @@ -52,19 +87,13 @@ def list_evaluations(function, offset=None, size=None, id=None, task=None, api_call += "/flow/%s" % ','.join([str(int(i)) for i in flow]) if uploader is not None: api_call += "/uploader/%s" % ','.join([str(int(i)) for i in uploader]) - if tag is not None: - api_call += "/tag/%s" % tag - return _list_evaluations(api_call) + return __list_evaluations(api_call) -def _list_evaluations(api_call): +def __list_evaluations(api_call): """Helper function to parse API calls which are lists of runs""" - try: - xml_string = _perform_api_call(api_call) - except OpenMLServerNoResult: - return dict() - + xml_string = _perform_api_call(api_call) evals_dict = xmltodict.parse(xml_string, force_list=('oml:evaluation',)) # Minimalistic check if the XML is useful if 'oml:evaluations' not in evals_dict: @@ -88,5 +117,4 @@ def _list_evaluations(api_call): eval_['oml:upload_time'], float(eval_['oml:value']), array_data) evals[run_id] = evaluation - return evals - + return evals \ No newline at end of file diff --git a/openml/flows/functions.py b/openml/flows/functions.py index 61a260f35..71d55d4d6 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -6,6 +6,7 @@ from openml._api_calls import _perform_api_call from openml.exceptions import OpenMLServerNoResult from . import OpenMLFlow +import openml.utils def get_flow(flow_id): @@ -30,8 +31,11 @@ def get_flow(flow_id): return flow -def list_flows(offset=None, size=None, tag=None): - """Return a list of all flows which are on OpenML. +def list_flows(offset=None, size=None, tag=None, **kwargs): + + """ + Return a list of all flows which are on OpenML. + (Supports large amount of results) Parameters ---------- @@ -41,6 +45,8 @@ def list_flows(offset=None, size=None, tag=None): the maximum number of flows to return tag : str, optional the tag to include + kwargs: dict, optional + Legal filter operators: uploader. Returns ------- @@ -57,17 +63,29 @@ def list_flows(offset=None, size=None, tag=None): - external version - uploader """ - api_call = "flow/list" - if offset is not None: - api_call += "/offset/%d" % int(offset) + return openml.utils.list_all(_list_flows, offset=offset, size=size, tag=tag, **kwargs) + + +def _list_flows(**kwargs): + """ + Perform the api call that return a list of all flows. + + Parameters + ---------- + kwargs: dict, optional + Legal filter operators: uploader, tag, limit, offset. - if size is not None: - api_call += "/limit/%d" % int(size) + Returns + ------- + flows : dict + """ + api_call = "flow/list" - if tag is not None: - api_call += "/tag/%s" % tag + if kwargs is not None: + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) - return _list_flows(api_call) + return __list_flows(api_call) def flow_exists(name, external_version): @@ -79,7 +97,7 @@ def flow_exists(name, external_version): ---------- name : string Name of the flow - version : string + external_version : string Version information associated with flow. Returns @@ -108,11 +126,9 @@ def flow_exists(name, external_version): return False -def _list_flows(api_call): - try: - xml_string = _perform_api_call(api_call) - except OpenMLServerNoResult: - return dict() +def __list_flows(api_call): + + xml_string = _perform_api_call(api_call) flows_dict = xmltodict.parse(xml_string, force_list=('oml:flow',)) # Minimalistic check if the XML is useful @@ -186,11 +202,11 @@ def assert_flows_equal(flow1, flow2, # Tags aren't directly created by the server, # but the uploader has no control over them! 'tags'] - ignored_by_python_API = ['binary_url', 'binary_format', 'binary_md5', + ignored_by_python_api = ['binary_url', 'binary_format', 'binary_md5', 'model'] for key in set(flow1.__dict__.keys()).union(flow2.__dict__.keys()): - if key in generated_by_the_server + ignored_by_python_API: + if key in generated_by_the_server + ignored_by_python_api: continue attr1 = getattr(flow1, key, None) attr2 = getattr(flow2, key, None) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 44dcfec69..541d3dfa3 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -892,10 +892,11 @@ def _get_cached_run(run_id): def list_runs(offset=None, size=None, id=None, task=None, setup=None, - flow=None, uploader=None, tag=None, display_errors=False): - """List all runs matching all of the given filters. + flow=None, uploader=None, tag=None, display_errors=False, **kwargs): - Perform API call `/run/list/{filters} `_ + """ + List all runs matching all of the given filters. + (Supports large amount of results) Parameters ---------- @@ -919,17 +920,61 @@ def list_runs(offset=None, size=None, id=None, task=None, setup=None, display_errors : bool, optional (default=None) Whether to list runs which have an error (for example a missing prediction file). + + kwargs: dict, optional + Legal filter operators: task_type. + Returns ------- - list + dict + List of found runs. + """ + + return openml.utils.list_all(_list_runs, offset=offset, size=size, id=id, task=task, setup=setup, + flow=flow, uploader=uploader, tag=tag, display_errors=display_errors, **kwargs) + + +def _list_runs(id=None, task=None, setup=None, + flow=None, uploader=None, display_errors=False, **kwargs): + + """ + Perform API call `/run/list/{filters}' + ` + + Parameters + ---------- + The arguments that are lists are separated from the single value + ones which are put into the kwargs. + display_errors is also separated from the kwargs since it has a + default value. + + id : list, optional + + task : list, optional + + setup: list, optional + + flow : list, optional + + uploader : list, optional + + display_errors : bool, optional (default=None) + Whether to list runs which have an error (for example a missing + prediction file). + + kwargs: dict, optional + Legal filter operators: task_type. + + Returns + ------- + dict List of found runs. """ api_call = "run/list" - if offset is not None: - api_call += "/offset/%d" % int(offset) - if size is not None: - api_call += "/limit/%d" % int(size) + if kwargs is not None: + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) if id is not None: api_call += "/run/%s" % ','.join([str(int(i)) for i in id]) if task is not None: @@ -940,21 +985,14 @@ def list_runs(offset=None, size=None, id=None, task=None, setup=None, api_call += "/flow/%s" % ','.join([str(int(i)) for i in flow]) if uploader is not None: api_call += "/uploader/%s" % ','.join([str(int(i)) for i in uploader]) - if tag is not None: - api_call += "/tag/%s" % tag if display_errors: api_call += "/show_errors/true" + return __list_runs(api_call) - return _list_runs(api_call) - -def _list_runs(api_call): +def __list_runs(api_call): """Helper function to parse API calls which are lists of runs""" - try: - xml_string = _perform_api_call(api_call) - except OpenMLServerNoResult: - return dict() - + xml_string = _perform_api_call(api_call) runs_dict = xmltodict.parse(xml_string, force_list=('oml:run',)) # Minimalistic check if the XML is useful if 'oml:runs' not in runs_dict: @@ -984,4 +1022,4 @@ def _list_runs(api_call): runs[run_id] = run - return runs + return runs \ No newline at end of file diff --git a/openml/setups/functions.py b/openml/setups/functions.py index a78e07ae6..745da5a1e 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -9,6 +9,7 @@ from .setup import OpenMLSetup, OpenMLParameter from openml.flows import flow_exists from openml.exceptions import OpenMLServerNoResult +import openml.utils def setup_exists(flow, model=None): @@ -106,22 +107,40 @@ def get_setup(setup_id): return _create_setup_from_xml(result_dict) -def list_setups(flow=None, tag=None, setup=None, offset=None, size=None): - """List all setups matching all of the given filters. - - Perform API call `/setup/list/{filters}` +def list_setups(offset=None, size=None, flow=None, tag=None, setup=None): + """ + List all setups matching all of the given filters. Parameters ---------- + offset : int, optional + size : int, optional flow : int, optional - tag : str, optional - setup : list(int), optional - offset : int, optional + Returns + ------- + dict + """ - size : int, optional + return openml.utils.list_all(_list_setups, offset=offset, size=size, + flow=flow, tag=tag, setup=setup) + + +def _list_setups(setup=None, **kwargs): + """ + Perform API call `/setup/list/{filters}` + + Parameters + ---------- + The setup argument that is a list is separated from the single value + filters which are put into the kwargs. + + setup : list(int), optional + + kwargs: dict, optional + Legal filter operators: flow, setup, limit, offset, tag. Returns ------- @@ -129,28 +148,18 @@ def list_setups(flow=None, tag=None, setup=None, offset=None, size=None): """ api_call = "setup/list" - if offset is not None: - api_call += "/offset/%d" % int(offset) - if size is not None: - api_call += "/limit/%d" % int(size) if setup is not None: api_call += "/setup/%s" % ','.join([str(int(i)) for i in setup]) - if flow is not None: - api_call += "/flow/%s" % flow - if tag is not None: - api_call += "/tag/%s" % tag + if kwargs is not None: + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) - return _list_setups(api_call) + return __list_setups(api_call) -def _list_setups(api_call): +def __list_setups(api_call): """Helper function to parse API calls which are lists of setups""" - - try: - xml_string = openml._api_calls._perform_api_call(api_call) - except OpenMLServerNoResult: - return dict() - + xml_string = openml._api_calls._perform_api_call(api_call) setups_dict = xmltodict.parse(xml_string, force_list=('oml:setup',)) # Minimalistic check if the XML is useful if 'oml:setups' not in setups_dict: diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 32c4c0fec..e90c84ee1 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -12,7 +12,7 @@ from .task import OpenMLTask, _create_task_cache_dir from .. import config from .._api_calls import _perform_api_call - +import openml.utils def _get_cached_tasks(): tasks = OrderedDict() @@ -88,11 +88,16 @@ def _get_estimation_procedure_list(): return procs -def list_tasks(task_type_id=None, offset=None, size=None, tag=None): - """Return a number of tasks having the given tag and task_type_id +def list_tasks(task_type_id=None, offset=None, size=None, tag=None, **kwargs): + """ + Return a number of tasks having the given tag and task_type_id Parameters ---------- + Filter task_type_id is separated from the other filters because + it is used as task_type_id in the task description, but it is named + type when used as a filter in list tasks call. + task_type_id : int, optional ID of the task type as detailed `here `_. @@ -105,7 +110,6 @@ def list_tasks(task_type_id=None, offset=None, size=None, tag=None): - Machine Learning Challenge: 6 - Survival Analysis: 7 - Subgroup Discovery: 8 - offset : int, optional the number of tasks to skip, starting from the first size : int, optional @@ -113,6 +117,10 @@ def list_tasks(task_type_id=None, offset=None, size=None, tag=None): tag : str, optional the tag to include + kwargs: dict, optional + Legal filter operators: data_tag, status, data_id, data_name, number_instances, number_features, + number_classes, number_missing_values. + Returns ------- dict @@ -121,28 +129,54 @@ def list_tasks(task_type_id=None, offset=None, size=None, tag=None): task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned. """ - api_call = "task/list" - if task_type_id is not None: - api_call += "/type/%d" % int(task_type_id) + return openml.utils.list_all(_list_tasks, task_type_id=task_type_id, offset=offset, size=size, tag=tag, **kwargs) + + +def _list_tasks(task_type_id=None, **kwargs): + """ + Perform the api call to return a number of tasks having the given filters. + + Parameters + ---------- + Filter task_type_id is separated from the other filters because + it is used as task_type_id in the task description, but it is named + type when used as a filter in list tasks call. + + task_type_id : int, optional + ID of the task type as detailed + `here `_. - if offset is not None: - api_call += "/offset/%d" % int(offset) + - Supervised classification: 1 + - Supervised regression: 2 + - Learning curve: 3 + - Supervised data stream classification: 4 + - Clustering: 5 + - Machine Learning Challenge: 6 + - Survival Analysis: 7 + - Subgroup Discovery: 8 - if size is not None: - api_call += "/limit/%d" % int(size) + kwargs: dict, optional + Legal filter operators: tag, data_tag, status, limit, + offset, data_id, data_name, number_instances, number_features, + number_classes, number_missing_values. - if tag is not None: - api_call += "/tag/%s" % tag + Returns + ------- + dict + """ + api_call = "task/list" + if task_type_id is not None: + api_call += "/type/%d" % int(task_type_id) + if kwargs is not None: + for operator, value in kwargs.items(): + api_call += "/%s/%s" % (operator, value) + return __list_tasks(api_call) - return _list_tasks(api_call) +def __list_tasks(api_call): -def _list_tasks(api_call): - try: - xml_string = _perform_api_call(api_call) - except OpenMLServerNoResult: - return dict() - tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task','oml:input')) + xml_string = _perform_api_call(api_call) + tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task', 'oml:input')) # Minimalistic check if the XML is useful if 'oml:tasks' not in tasks_dict: raise ValueError('Error in return XML, does not contain "oml:runs": %s' diff --git a/openml/utils.py b/openml/utils.py index cc976b4c3..1ea725957 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -4,7 +4,6 @@ from openml.exceptions import OpenMLServerException - def extract_xml_tags(xml_tag_name, node, allow_none=True): """Helper to extract xml tags from xmltodict. @@ -43,7 +42,6 @@ def extract_xml_tags(xml_tag_name, node, allow_none=True): raise ValueError("Could not find tag '%s' in node '%s'" % (xml_tag_name, str(node))) - def _tag_entity(entity_type, entity_id, tag, untag=False): """Function that tags or untags a given entity on OpenML. As the OpenML API tag functions all consist of the same format, this function covers @@ -91,8 +89,8 @@ def _tag_entity(entity_type, entity_id, tag, untag=False): # no tags, return empty list return [] - -def list_all(listing_call, batch_size=10000, *args, **filters): + +def list_all(listing_call, *args, **filters): """Helper to handle paged listing requests. Example usage: @@ -106,8 +104,6 @@ def list_all(listing_call, batch_size=10000, *args, **filters): ---------- listing_call : callable Call listing, e.g. list_evaluations. - batch_size : int (default: 10000) - Batch size for paging. *args : Variable length argument list Any required arguments for the listing call. **filters : Arbitrary keyword arguments @@ -117,17 +113,34 @@ def list_all(listing_call, batch_size=10000, *args, **filters): ------- dict """ + + # default batch size per paging. + batch_size = 10000 + # eliminate filters that have a None value + active_filters = {key: value for key, value in filters.items() if value is not None} page = 0 - has_more = 1 result = {} - - while has_more: + # max number of results to be shown + limit = None + offset = 0 + cycle = True + if 'size' in active_filters: + limit = active_filters['size'] + del active_filters['size'] + # check if the batch size is greater than the number of results that need to be returned. + if limit is not None: + if batch_size > limit: + batch_size = limit + if 'offset' in active_filters: + offset = active_filters['offset'] + del active_filters['offset'] + while cycle: try: new_batch = listing_call( *args, - size=batch_size, - offset=batch_size*page, - **filters + limit=batch_size, + offset=offset + batch_size * page, + **active_filters ) except OpenMLServerException as e: if page == 0 and e.args[0] == 'No results': @@ -136,6 +149,13 @@ def list_all(listing_call, batch_size=10000, *args, **filters): break result.update(new_batch) page += 1 - has_more = (len(new_batch) == batch_size) + if limit is not None: + limit -= batch_size + # check if the number of required results has been achieved + if limit == 0: + break + # check if there are enough results to fulfill a batch + if limit < batch_size: + batch_size = limit - return result + return result \ No newline at end of file diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 85986fdf1..83ceffa7f 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -139,6 +139,11 @@ def test_list_datasets_by_tag(self): self.assertGreaterEqual(len(datasets), 100) self._check_datasets(datasets) + def test_list_datasets_by_size(self): + datasets = openml.datasets.list_datasets(size=10050) + self.assertGreaterEqual(len(datasets), 120) + self._check_datasets(datasets) + def test_list_datasets_by_number_instances(self): datasets = openml.datasets.list_datasets(number_instances="5..100") self.assertGreaterEqual(len(datasets), 4) @@ -169,7 +174,7 @@ def test_list_datasets_paginate(self): max = 100 for i in range(0, max, size): datasets = openml.datasets.list_datasets(offset=i, size=size) - self.assertGreaterEqual(size, len(datasets)) + self.assertEqual(size, len(datasets)) self._check_datasets(datasets) def test_list_datasets_empty(self): diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 1e362014e..d28a834b3 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -918,7 +918,11 @@ def test_get_runs_list_by_filters(self): uploaders_2 = [29, 274] flows = [74, 1718] - self.assertRaises(openml.exceptions.OpenMLServerError, openml.runs.list_runs) + ''' + Since the results are taken by batch size, the function does not throw an OpenMLServerError anymore. + Instead it throws a TimeOutException. For the moment commented out. + ''' + #self.assertRaises(openml.exceptions.OpenMLServerError, openml.runs.list_runs) runs = openml.runs.list_runs(id=ids) self.assertEqual(len(runs), 2) From 5058e1d029ffa6500889734a4c4e22435c835d73 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Wed, 28 Mar 2018 12:26:04 -0400 Subject: [PATCH 71/86] add string representation for runs (#391) --- openml/runs/run.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 4a73999d8..4fa7c62b2 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -9,7 +9,7 @@ import openml from ..tasks import get_task -from .._api_calls import _perform_api_call, _file_id_to_url, _read_url_files +from .._api_calls import _perform_api_call, _file_id_to_url from ..exceptions import PyOpenMLError @@ -54,6 +54,17 @@ def __init__(self, task_id, flow_id, dataset_id, setup_string=None, self.tags = tags self.predictions_url = predictions_url + def __str__(self): + flow_name = self.flow_name + if len(flow_name) > 26: + # long enough to show sklearn.pipeline.Pipeline + flow_name = flow_name[:26] + "..." + return "[run id: {}, task id: {}, flow id: {}, flow name: {}]".format( + self.run_id, self.task_id, self.flow_id, flow_name) + + def _repr_pretty_(self, pp, cycle): + pp.text(str(self)) + def _generate_arff_dict(self): """Generates the arff dictionary for uploading predictions to the server. From 5b701bb76d081a7f7d0d90bcd64055965a932af0 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 29 Mar 2018 10:18:43 +0200 Subject: [PATCH 72/86] ADD unit test to ensure example listing (#421) * ADD unit test to ensure example listing * Update test_study_examples.py --- tests/test_study/test_study_examples.py | 53 +++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/test_study/test_study_examples.py diff --git a/tests/test_study/test_study_examples.py b/tests/test_study/test_study_examples.py new file mode 100644 index 000000000..1dea4085c --- /dev/null +++ b/tests/test_study/test_study_examples.py @@ -0,0 +1,53 @@ +from openml.testing import TestBase + + +class TestStudyFunctions(TestBase): + _multiprocess_can_split_ = True + """Test the example code of Bischl et al. (2018)""" + + def test_Figure1a(self): + """Test listing in Figure 1a on a single task and the old OpenML100 study. + + The original listing is pasted into the comment below because it the + actual unit test differs a bit, as for example it does not run for all tasks, + but only a single one. + + import openml + import sklearn.tree, sklearn.preprocessing + benchmark_suite = openml.study.get_study('OpenML-CC18','tasks') # obtain the benchmark suite + clf = sklearn.pipeline.Pipeline(steps=[('imputer',sklearn.preprocessing.Imputer()), ('estimator',sklearn.tree.DecisionTreeClassifier())]) # build a sklearn classifier + for task_id in benchmark_suite.tasks: # iterate over all tasks + task = openml.tasks.get_task(task_id) # download the OpenML task + X, y = task.get_X_and_y() # get the data (not used in this example) + openml.config.apikey = 'FILL_IN_OPENML_API_KEY' # set the OpenML Api Key + run = openml.runs.run_model_on_task(task,clf) # run classifier on splits (requires API key) + score = run.get_metric_fn(sklearn.metrics.accuracy_score) # print accuracy score + print('Data set: %s; Accuracy: %0.2f' % (task.get_dataset().name,score.mean())) + run.publish() # publish the experiment on OpenML (optional) + print('URL for run: %s/run/%d' %(openml.config.server,run.run_id)) + """ + import openml + import sklearn.tree, sklearn.preprocessing + benchmark_suite = openml.study.get_study( + 'OpenML100', 'tasks' + ) # obtain the benchmark suite + clf = sklearn.pipeline.Pipeline( + steps=[ + ('imputer', sklearn.preprocessing.Imputer()), + ('estimator', sklearn.tree.DecisionTreeClassifier()) + ] + ) # build a sklearn classifier + for task_id in benchmark_suite.tasks[:1]: # iterate over all tasks + task = openml.tasks.get_task(task_id) # download the OpenML task + X, y = task.get_X_and_y() # get the data (not used in this example) + openml.config.apikey = openml.config.apikey # set the OpenML Api Key + run = openml.runs.run_model_on_task( + task, clf, + ) # run classifier on splits (requires API key) + score = run.get_metric_fn( + sklearn.metrics.accuracy_score + ) # print accuracy score + print('Data set: %s; Accuracy: %0.2f' % ( + task.get_dataset().name, score.mean())) + run.publish() # publish the experiment on OpenML (optional) + print('URL for run: %s/run/%d' % (openml.config.server, run.run_id)) From babe8a608ad0679510a642a05f3a207ce921af00 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 29 Mar 2018 10:46:45 +0200 Subject: [PATCH 73/86] improve error message (#420) --- openml/datasets/dataset.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 8761837eb..675a7c4ba 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -151,8 +151,10 @@ def remove_tag(self, tag): def __eq__(self, other): if type(other) != OpenMLDataset: return False - elif self.id == other._id or \ - (self.name == other._name and self.version == other._version): + elif ( + self.dataset_id == other.dataset_id + or (self.name == other._name and self.version == other._version) + ): return True else: return False @@ -222,7 +224,10 @@ def get_data(self, target=None, rval = [] if not self._data_features_supported(): - raise PyOpenMLError('Dataset not compatible, PyOpenML cannot handle string features') + raise PyOpenMLError( + 'Dataset %d not compatible, PyOpenML cannot handle string ' + 'features' % self.dataset_id + ) path = self.data_pickle_file if not os.path.exists(path): From d06c4b9677bcdb0a1ca269f8a805fbece60f9ea9 Mon Sep 17 00:00:00 2001 From: William Raynaut Date: Thu, 29 Mar 2018 13:56:11 +0200 Subject: [PATCH 74/86] Added appveyor files (#362) --- appveyor.yml | 52 +++++++++++++++++++++++ appveyor/run_with_env.cmd | 88 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 appveyor.yml create mode 100644 appveyor/run_with_env.cmd diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..e89e6fc7d --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,52 @@ + +environment: + global: + CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\scikit-learn-contrib\\run_with_env.cmd" + + matrix: + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + PYTHON_ARCH: "64" + MINICONDA: "C:\\Miniconda35-x64" + + - PYTHON: "C:\\Python35" + PYTHON_VERSION: "3.5" + PYTHON_ARCH: "32" + MINICONDA: "C:\\Miniconda35" + +matrix: + fast_finish: true + + +install: + # Miniconda is pre-installed in the worker build + - "SET PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" + - "python -m pip install -U pip" + + # Check that we have the expected version and architecture for Python + - "python --version" + - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" + - "pip --version" + + # Remove cygwin because it clashes with conda + # see http://help.appveyor.com/discussions/problems/3712-git-remote-https-seems-to-be-broken + - rmdir C:\\cygwin /s /q + + # Update previous packages and install the build and runtime dependencies of the project. + # XXX: setuptools>23 is currently broken on Win+py3 with numpy + # (https://github.com/pypa/setuptools/issues/728) + - conda update --all --yes setuptools=23 + + # Install the build and runtime dependencies of the project. + - "cd C:\\projects\\openml-python" + - conda install --quiet --yes mock numpy scipy nose requests scikit-learn nbformat python-dateutil nbconvert + - pip install liac-arff xmltodict oslo.concurrency + - "%CMD_IN_ENV% python setup.py install" + + +# Not a .NET project, we build scikit-learn in the install step instead +build: false + +test_script: + - "cd C:\\projects\\openml-python" + - "%CMD_IN_ENV% python setup.py test" diff --git a/appveyor/run_with_env.cmd b/appveyor/run_with_env.cmd new file mode 100644 index 000000000..5da547c49 --- /dev/null +++ b/appveyor/run_with_env.cmd @@ -0,0 +1,88 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific +:: environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +:: +:: Notes about batch files for Python people: +:: +:: Quotes in values are literally part of the values: +:: SET FOO="bar" +:: FOO is now five characters long: " b a r " +:: If you don't want quotes, don't include them on the right-hand side. +:: +:: The CALL lines at the end of this file look redundant, but if you move them +:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y +:: case, I don't know why. +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows +SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf + +:: Extract the major and minor versions, and allow for the minor version to be +:: more than 9. This requires the version number to have two dots in it. +SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% +IF "%PYTHON_VERSION:~3,1%" == "." ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% +) ELSE ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% +) + +:: Based on the Python version, determine what SDK version to use, and whether +:: to set the SDK for 64-bit. +IF %MAJOR_PYTHON_VERSION% == 2 ( + SET WINDOWS_SDK_VERSION="v7.0" + SET SET_SDK_64=Y +) ELSE ( + IF %MAJOR_PYTHON_VERSION% == 3 ( + SET WINDOWS_SDK_VERSION="v7.1" + IF %MINOR_PYTHON_VERSION% LEQ 4 ( + SET SET_SDK_64=Y + ) ELSE ( + SET SET_SDK_64=N + IF EXIST "%WIN_WDK%" ( + :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ + REN "%WIN_WDK%" 0wdf + ) + ) + ) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 + ) +) + +IF %PYTHON_ARCH% == 64 ( + IF %SET_SDK_64% == Y ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) ELSE ( + ECHO Using default MSVC build environment for 64 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) From fb20762aa2c994fba46ede90ec86bf07d8495457 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 29 Mar 2018 17:35:11 +0200 Subject: [PATCH 75/86] MAINT bump required liac-arff version (#427) * MAINT bump required liac-arff version * Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f928fe964..ad1e12cf0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ mock numpy>=1.6.2 scipy>=0.13.3 -liac-arff>=2.1.1 +liac-arff>=2.2.1 xmltodict nose requests From 074e0cf0993ff9d1bb679a688ac05cbaa39473ba Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Tue, 3 Apr 2018 06:50:32 -0400 Subject: [PATCH 76/86] fixes locking bug ? (#429) --- openml/tasks/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index e90c84ee1..cf9e0a2a3 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -278,7 +278,7 @@ def get_task(task_id): tid_cache_dir = _create_task_cache_dir(task_id) with lockutils.external_lock( - name='datasets.functions.get_dataset:%d' % task_id, + name='task.functions.get_task:%d' % task_id, lock_path=os.path.join(config.get_cache_directory(), 'locks'), ): try: From 6ac98aaf0f5d10b4d6f8e737dd975640f00882f1 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 4 Apr 2018 10:14:13 +0200 Subject: [PATCH 77/86] Add contributing information (#368) * ADD contributing guidelines and github templates * Update CONTRIBUTING.md * ADD reference to scikit-learn to the license * Added clarification for unit test on Windows. --- CONTRIBUTING.md | 178 +++++++++++++++++++++++++++++++++++++++ ISSUE_TEMPLATE.md | 33 ++++++++ LICENSE | 40 ++++++++- PULL_REQUEST_TEMPLATE.md | 21 +++++ 4 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING.md create mode 100644 ISSUE_TEMPLATE.md create mode 100644 PULL_REQUEST_TEMPLATE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..2a215a985 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,178 @@ +How to contribute +----------------- + +The preferred workflow for contributing to the OpenML python connector is to +fork the [main repository](https://github.com/openml/openml-python) on +GitHub, clone, check out the branch `develop`, and develop on a new branch +branch. Steps: + +1. Fork the [project repository](https://github.com/openml/openml-python) + by clicking on the 'Fork' button near the top right of the page. This creates + a copy of the code under your GitHub user account. For more details on + how to fork a repository see [this guide](https://help.github.com/articles/fork-a-repo/). + +2. Clone your fork of the openml-python repo from your GitHub account to your +local disk: + + ```bash + $ git clone git@github.com:YourLogin/openml-python.git + $ cd openml-python + ``` + +3. Swith to the ``develop`` branch: + + ```bash + $ git checkout develop + ``` + +3. Create a ``feature`` branch to hold your development changes: + + ```bash + $ git checkout -b feature/my-feature + ``` + + Always use a ``feature`` branch. It's good practice to never work on the ``master`` or ``develop`` branch! To make the nature of your pull request easily visible, please perpend the name of the branch with the type of changes you want to merge, such as ``feature`` if it contains a new feature, ``fix`` for a bugfix, ``doc`` for documentation and ``maint`` for other maintenance on the package. + +4. Develop the feature on your feature branch. Add changed files using ``git add`` and then ``git commit`` files: + + ```bash + $ git add modified_files + $ git commit + ``` + + to record your changes in Git, then push the changes to your GitHub account with: + + ```bash + $ git push -u origin my-feature + ``` + +5. Follow [these instructions](https://help.github.com/articles/creating-a-pull-request-from-a-fork) +to create a pull request from your fork. This will send an email to the committers. + +(If any of the above seems like magic to you, please look up the +[Git documentation](https://git-scm.com/documentation) on the web, or ask a friend or another contributor for help.) + +Pull Request Checklist +---------------------- + +We recommended that your contribution complies with the +following rules before you submit a pull request: + +- Follow the + [pep8 style guilde](https://www.python.org/dev/peps/pep-0008/). + +- If your pull request addresses an issue, please use the pull request title + to describe the issue and mention the issue number in the pull request description. This will make sure a link back to the original issue is + created. + +- An incomplete contribution -- where you expect to do more work before + receiving a full review -- should be prefixed `[WIP]` (to indicate a work + in progress) and changed to `[MRG]` when it matures. WIPs may be useful + to: indicate you are working on something to avoid duplicated work, + request broad review of functionality or API, or seek collaborators. + WIPs often benefit from the inclusion of a + [task list](https://github.com/blog/1375-task-lists-in-gfm-issues-pulls-comments) + in the PR description. + +- All tests pass when running `nosetests`. On + Unix-like systems, check with (from the toplevel source folder): + + ```bash + $ nosetests + ``` + + For Windows systems, execute the command from an Anaconda Prompt or add `nosetests` to PATH before executing the command. + +- Documentation and high-coverage tests are necessary for enhancements to be + accepted. Bug-fixes or new features should be provided with + [non-regression tests](https://en.wikipedia.org/wiki/Non-regression_testing). + These tests verify the correct behavior of the fix or feature. In this + manner, further modifications on the code base are granted to be consistent + with the desired behavior. + For the Bug-fixes case, at the time of the PR, this tests should fail for + the code base in develop and pass for the PR code. + + +You can also check for common programming errors with the following +tools: + +- Code with good unittest **coverage** (at least 80%), check with: + + ```bash + $ pip install nose coverage + $ nosetests --with-coverage path/to/tests_for_package + ``` + +- No pyflakes warnings, check with: + + ```bash + $ pip install pyflakes + $ pyflakes path/to/module.py + ``` + +- No PEP8 warnings, check with: + + ```bash + $ pip install pep8 + $ pep8 path/to/module.py + ``` + +Filing bugs +----------- +We use GitHub issues to track all bugs and feature requests; feel free to +open an issue if you have found a bug or wish to see a feature implemented. + +It is recommended to check that your issue complies with the +following rules before submitting: + +- Verify that your issue is not being currently addressed by other + [issues](https://github.com/openml/openml-python/issues) + or [pull requests](https://github.com/openml/openml-python/pulls). + +- Please ensure all code snippets and error messages are formatted in + appropriate code blocks. + See [Creating and highlighting code blocks](https://help.github.com/articles/creating-and-highlighting-code-blocks). + +- Please include your operating system type and version number, as well + as your Python, openml, scikit-learn, numpy, and scipy versions. This information + can be found by running the following code snippet: + + ```python + import platform; print(platform.platform()) + import sys; print("Python", sys.version) + import numpy; print("NumPy", numpy.__version__) + import scipy; print("SciPy", scipy.__version__) + import sklearn; print("Scikit-Learn", sklearn.__version__) + import openml; print("OpenML", openml.__version__) + ``` + +New contributor tips +-------------------- + +A great way to start contributing to scikit-learn is to pick an item +from the list of [Easy issues](https://github.com/openml/openml-python/issues?q=label%3Aeasy) +in the issue tracker. Resolving these issues allow you to start +contributing to the project without much prior knowledge. Your +assistance in this area will be greatly appreciated by the more +experienced developers as it helps free up their time to concentrate on +other issues. + +Documentation +------------- + +We are glad to accept any sort of documentation: function docstrings, +reStructuredText documents (like this one), tutorials, etc. +reStructuredText documents live in the source code repository under the +doc/ directory. + +You can edit the documentation using any text editor and then generate +the HTML output by typing ``make html`` from the doc/ directory. +The resulting HTML files will be placed in ``build/html/`` and are viewable in +a web browser. See the ``README`` file in the ``doc/`` directory for more +information. + +For building the documentation, you will need +[sphinx](http://sphinx.pocoo.org/), +[matplotlib](http://matplotlib.org/), and +[pillow](http://pillow.readthedocs.io/en/latest/). +[sphinx-bootstrap-theme](https://ryan-roemer.github.io/sphinx-bootstrap-theme/) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..bcd5e0c1e --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,33 @@ +#### Description + + +#### Steps/Code to Reproduce + + +#### Expected Results + + +#### Actual Results + + +#### Versions + + + + \ No newline at end of file diff --git a/LICENSE b/LICENSE index 924b5b561..146b8cc36 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2014-2017, Matthias Feurer, Jan van Rijn, Andreas Müller, +Copyright (c) 2014-2018, Matthias Feurer, Jan van Rijn, Andreas Müller, Joaquin Vanschoren and others. All rights reserved. @@ -28,3 +28,41 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License of the files CONTRIBUTING.md, ISSUE_TEMPLATE.md and +PULL_REQUEST_TEMPLATE.md: + +Those files are modifications of the respecting templates in scikit-learn and +they are licensed under a New BSD license: + +New BSD License + +Copyright (c) 2007–2018 The scikit-learn developers. +All rights reserved. + + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of the Scikit-learn Developers nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..c73beebea --- /dev/null +++ b/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ + +#### Reference Issue + + + +#### What does this PR implement/fix? Explain your changes. + + +#### How should this PR be tested? + + +#### Any other comments? + From 3232d0d5a64ebb46569298ddc8f702878b67f9bd Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 4 Apr 2018 13:16:35 +0200 Subject: [PATCH 78/86] Improve error logging (#428) * ADD hash checks to datasets * split target on comma, allows errors on multiple targets * FIX unit test --- openml/datasets/dataset.py | 5 ++++- openml/datasets/functions.py | 11 +++++++---- openml/exceptions.py | 5 +++++ tests/test_datasets/test_dataset_functions.py | 4 ++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 675a7c4ba..f7b86888c 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -270,7 +270,10 @@ def get_data(self, target=None, rval.append(data) else: if isinstance(target, six.string_types): - target = [target] + if ',' in target: + target = target.split(',') + else: + target = [target] targets = np.array([True if column in target else False for column in attribute_names]) if np.sum(targets) > 1: diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 6e3123bce..ecb5c2674 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -10,7 +10,8 @@ import openml.utils from .dataset import OpenMLDataset -from ..exceptions import OpenMLCacheException, OpenMLServerNoResult +from ..exceptions import OpenMLCacheException, OpenMLServerNoResult, \ + OpenMLHashException from .. import config from .._api_calls import _perform_api_call, _read_url @@ -404,12 +405,14 @@ def _get_dataset_arff(did_cache_dir, description): url = description['oml:url'] arff_string = _read_url(url) md5 = hashlib.md5() - md5.update(arff_string.encode('utf8')) + md5.update(arff_string.encode('utf-8')) md5_checksum = md5.hexdigest() if md5_checksum != md5_checksum_fixture: - raise ValueError( + raise OpenMLHashException( 'Checksum %s of downloaded dataset %d is unequal to the checksum ' - '%s sent by the server.' % (md5_checksum, did, md5_checksum_fixture) + '%s sent by the server.' % ( + md5_checksum, int(did), md5_checksum_fixture + ) ) with io.open(output_file_path, "w", encoding='utf8') as fh: diff --git a/openml/exceptions.py b/openml/exceptions.py index 386e25cdc..e7df0708d 100644 --- a/openml/exceptions.py +++ b/openml/exceptions.py @@ -35,3 +35,8 @@ class OpenMLCacheException(PyOpenMLError): """Dataset / task etc not found in cache""" def __init__(self, message): super(OpenMLCacheException, self).__init__(message) + + +class OpenMLHashException(PyOpenMLError): + """Locally computed hash is different than hash announced by the server.""" + pass \ No newline at end of file diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 83ceffa7f..9469bcb1b 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -17,7 +17,7 @@ import openml from openml import OpenMLDataset -from openml.exceptions import OpenMLCacheException, PyOpenMLError +from openml.exceptions import OpenMLCacheException, PyOpenMLError, OpenMLHashException from openml.testing import TestBase from openml.utils import _tag_entity @@ -268,7 +268,7 @@ def test__getarff_md5_issue(self): 'oml:url': 'https://www.openml.org/data/download/61', } self.assertRaisesRegexp( - ValueError, + OpenMLHashException, 'Checksum ad484452702105cbf3d30f8deaba39a9 of downloaded dataset 5 ' 'is unequal to the checksum abc sent by the server.', _get_dataset_arff, From c942308787b0b2d156625bd9e275696ef639147a Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 6 Apr 2018 11:03:56 +0200 Subject: [PATCH 79/86] Fix/#392 (#430) * MAINT move from cell executor to exec to test jupyter notebooks * Mock calls to /run/upload/ in notebook unit tests * FIX mock patch target * Change backend --- examples/OpenML_Tutorial.ipynb | 18 ++++--- openml/datasets/dataset.py | 13 +++-- openml/datasets/functions.py | 11 +++-- openml/evaluations/functions.py | 6 +-- openml/exceptions.py | 4 ++ openml/flows/flow.py | 11 +++-- openml/flows/functions.py | 14 +++--- openml/runs/functions.py | 10 ++-- openml/runs/run.py | 9 ++-- openml/study/functions.py | 6 +-- openml/tasks/functions.py | 8 ++-- openml/tasks/task.py | 7 +-- openml/utils.py | 6 +-- tests/test_examples/test_OpenMLDemo.py | 66 +++++++++++++++++--------- tests/test_flows/test_flow.py | 2 +- 15 files changed, 116 insertions(+), 75 deletions(-) diff --git a/examples/OpenML_Tutorial.ipynb b/examples/OpenML_Tutorial.ipynb index d670a6ead..a8ec24e78 100644 --- a/examples/OpenML_Tutorial.ipynb +++ b/examples/OpenML_Tutorial.ipynb @@ -23,12 +23,18 @@ ] }, { - "cell_type": "raw", - "metadata": {}, + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, "source": [ - "# Install OpenML (developer version)\n", - "# 'pip install openml' coming up (october 2017) \n", - "pip install git+https://github.com/openml/openml-python.git@develop" + "# Installation\n", + "\n", + "* Up to now: `pip install git+https://github.com/openml/openml-python.git@develop`\n", + "* In the future: `pip install openml`\n", + "* Check out the installation guide: [https://openml.github.io/openml-python/stable/#installation](https://openml.github.io/openml-python/stable/#installation)" ] }, { @@ -1547,7 +1553,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.1" + "version": "3.6.2" } }, "nbformat": 4, diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index f7b86888c..f25557783 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -13,7 +13,7 @@ from .data_feature import OpenMLDataFeature from ..exceptions import PyOpenMLError -from .._api_calls import _perform_api_call +import openml._api_calls logger = logging.getLogger(__name__) @@ -135,7 +135,7 @@ def push_tag(self, tag): Tag to attach to the dataset. """ data = {'data_id': self.dataset_id, 'tag': tag} - _perform_api_call("/data/tag", data=data) + openml._api_calls._perform_api_call("/data/tag", data=data) def remove_tag(self, tag): """Removes a tag from this dataset on the server. @@ -146,7 +146,7 @@ def remove_tag(self, tag): Tag to attach to the dataset. """ data = {'data_id': self.dataset_id, 'tag': tag} - _perform_api_call("/data/untag", data=data) + openml._api_calls._perform_api_call("/data/untag", data=data) def __eq__(self, other): if type(other) != OpenMLDataset: @@ -432,8 +432,11 @@ def publish(self): if self.data_file is not None: file_dictionary['dataset'] = self.data_file - return_value = _perform_api_call("/data/", file_dictionary=file_dictionary, - file_elements=file_elements) + return_value = openml._api_calls._perform_api_call( + "/data/", + file_dictionary=file_dictionary, + file_elements=file_elements, + ) self.dataset_id = int(xmltodict.parse(return_value)['oml:upload_data_set']['oml:id']) return self diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index ecb5c2674..fa6e235b0 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -9,11 +9,12 @@ import xmltodict import openml.utils +import openml._api_calls from .dataset import OpenMLDataset from ..exceptions import OpenMLCacheException, OpenMLServerNoResult, \ OpenMLHashException from .. import config -from .._api_calls import _perform_api_call, _read_url +from .._api_calls import _read_url ############################################################################ @@ -206,7 +207,7 @@ def _list_datasets(**kwargs): def __list_datasets(api_call): - xml_string = _perform_api_call(api_call) + xml_string = openml._api_calls._perform_api_call(api_call) datasets_dict = xmltodict.parse(xml_string, force_list=('oml:dataset',)) # Minimalistic check if the XML is useful @@ -357,7 +358,7 @@ def _get_dataset_description(did_cache_dir, dataset_id): try: return _get_cached_dataset_description(dataset_id) except (OpenMLCacheException): - dataset_xml = _perform_api_call("data/%d" % dataset_id) + dataset_xml = openml._api_calls._perform_api_call("data/%d" % dataset_id) with io.open(description_file, "w", encoding='utf8') as fh: fh.write(dataset_xml) @@ -450,7 +451,7 @@ def _get_dataset_features(did_cache_dir, dataset_id): with io.open(features_file, encoding='utf8') as fh: features_xml = fh.read() except (OSError, IOError): - features_xml = _perform_api_call("data/features/%d" % dataset_id) + features_xml = openml._api_calls._perform_api_call("data/features/%d" % dataset_id) with io.open(features_file, "w", encoding='utf8') as fh: fh.write(features_xml) @@ -486,7 +487,7 @@ def _get_dataset_qualities(did_cache_dir, dataset_id): with io.open(qualities_file, encoding='utf8') as fh: qualities_xml = fh.read() except (OSError, IOError): - qualities_xml = _perform_api_call("data/qualities/%d" % dataset_id) + qualities_xml = openml._api_calls._perform_api_call("data/qualities/%d" % dataset_id) with io.open(qualities_file, "w", encoding='utf8') as fh: fh.write(qualities_xml) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 9711fd574..115455a12 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -2,7 +2,7 @@ from openml.exceptions import OpenMLServerNoResult import openml.utils -from .._api_calls import _perform_api_call +import openml._api_calls from ..evaluations import OpenMLEvaluation @@ -93,7 +93,7 @@ def _list_evaluations(function, id=None, task=None, def __list_evaluations(api_call): """Helper function to parse API calls which are lists of runs""" - xml_string = _perform_api_call(api_call) + xml_string = openml._api_calls._perform_api_call(api_call) evals_dict = xmltodict.parse(xml_string, force_list=('oml:evaluation',)) # Minimalistic check if the XML is useful if 'oml:evaluations' not in evals_dict: @@ -117,4 +117,4 @@ def __list_evaluations(api_call): eval_['oml:upload_time'], float(eval_['oml:value']), array_data) evals[run_id] = evaluation - return evals \ No newline at end of file + return evals diff --git a/openml/exceptions.py b/openml/exceptions.py index e7df0708d..c162485d5 100644 --- a/openml/exceptions.py +++ b/openml/exceptions.py @@ -25,6 +25,10 @@ def __init__(self, message, code=None, additional=None, url=None): self.url = url super(OpenMLServerException, self).__init__(message) + def __str__(self): + return '%s returned code %s: %s' % ( + self.url, self.code, self.message, + ) class OpenMLServerNoResult(OpenMLServerException): """exception for when the result of the server is empty. """ diff --git a/openml/flows/flow.py b/openml/flows/flow.py index 15fd1c8ef..30f0b4b22 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -3,7 +3,7 @@ import six import xmltodict -from .._api_calls import _perform_api_call +import openml._api_calls from ..utils import extract_xml_tags @@ -341,7 +341,10 @@ def publish(self): xml_description = self._to_xml() file_elements = {'description': xml_description} - return_value = _perform_api_call("flow/", file_elements=file_elements) + return_value = openml._api_calls._perform_api_call( + "flow/", + file_elements=file_elements, + ) flow_id = int(xmltodict.parse(return_value)['oml:upload_flow']['oml:id']) flow = openml.flows.functions.get_flow(flow_id) _copy_server_fields(flow, self) @@ -364,7 +367,7 @@ def push_tag(self, tag): Tag to attach to the flow. """ data = {'flow_id': self.flow_id, 'tag': tag} - _perform_api_call("/flow/tag", data=data) + openml._api_calls._perform_api_call("/flow/tag", data=data) def remove_tag(self, tag): """Removes a tag from this flow on the server. @@ -375,7 +378,7 @@ def remove_tag(self, tag): Tag to attach to the flow. """ data = {'flow_id': self.flow_id, 'tag': tag} - _perform_api_call("/flow/untag", data=data) + openml._api_calls._perform_api_call("/flow/untag", data=data) def _copy_server_fields(source_flow, target_flow): diff --git a/openml/flows/functions.py b/openml/flows/functions.py index 71d55d4d6..35bbcfd1a 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -3,8 +3,7 @@ import xmltodict import six -from openml._api_calls import _perform_api_call -from openml.exceptions import OpenMLServerNoResult +import openml._api_calls from . import OpenMLFlow import openml.utils @@ -23,7 +22,7 @@ def get_flow(flow_id): except: raise ValueError("Flow ID must be an int, got %s." % str(flow_id)) - flow_xml = _perform_api_call("flow/%d" % flow_id) + flow_xml = openml._api_calls._perform_api_call("flow/%d" % flow_id) flow_dict = xmltodict.parse(flow_xml) flow = OpenMLFlow._from_dict(flow_dict) @@ -114,9 +113,10 @@ def flow_exists(name, external_version): if not (isinstance(name, six.string_types) and len(external_version) > 0): raise ValueError('Argument \'version\' should be a non-empty string') - xml_response = _perform_api_call( - "flow/exists", data={'name': name, 'external_version': - external_version}) + xml_response = openml._api_calls._perform_api_call( + "flow/exists", + data={'name': name, 'external_version': external_version}, + ) result_dict = xmltodict.parse(xml_response) flow_id = int(result_dict['oml:flow_exists']['oml:id']) @@ -128,7 +128,7 @@ def flow_exists(name, external_version): def __list_flows(api_call): - xml_string = _perform_api_call(api_call) + xml_string = openml._api_calls._perform_api_call(api_call) flows_dict = xmltodict.parse(xml_string, force_list=('oml:flow',)) # Minimalistic check if the XML is useful diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 541d3dfa3..5190797c7 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -14,13 +14,13 @@ import openml import openml.utils +import openml._api_calls from ..exceptions import PyOpenMLError, OpenMLServerNoResult from .. import config from ..flows import sklearn_to_flow, get_flow, flow_exists, _check_n_jobs, \ _copy_server_fields from ..setups import setup_exists, initialize_model from ..exceptions import OpenMLCacheException, OpenMLServerException -from .._api_calls import _perform_api_call from .run import OpenMLRun, _get_version_information from .trace import OpenMLRunTrace, OpenMLTraceIteration @@ -150,7 +150,7 @@ def get_run_trace(run_id): openml.runs.OpenMLTrace """ - trace_xml = _perform_api_call('run/trace/%d' % run_id) + trace_xml = openml._api_calls._perform_api_call('run/trace/%d' % run_id) run_trace = _create_trace_from_description(trace_xml) return run_trace @@ -653,7 +653,7 @@ def get_run(run_id): return _get_cached_run(run_id) except (OpenMLCacheException): - run_xml = _perform_api_call("run/%d" % run_id) + run_xml = openml._api_calls._perform_api_call("run/%d" % run_id) with io.open(run_file, "w", encoding='utf8') as fh: fh.write(run_xml) @@ -992,7 +992,7 @@ def _list_runs(id=None, task=None, setup=None, def __list_runs(api_call): """Helper function to parse API calls which are lists of runs""" - xml_string = _perform_api_call(api_call) + xml_string = openml._api_calls._perform_api_call(api_call) runs_dict = xmltodict.parse(xml_string, force_list=('oml:run',)) # Minimalistic check if the XML is useful if 'oml:runs' not in runs_dict: @@ -1022,4 +1022,4 @@ def __list_runs(api_call): runs[run_id] = run - return runs \ No newline at end of file + return runs diff --git a/openml/runs/run.py b/openml/runs/run.py index 4fa7c62b2..7a01433c5 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -8,8 +8,9 @@ import xmltodict import openml +import openml._api_calls from ..tasks import get_task -from .._api_calls import _perform_api_call, _file_id_to_url +from .._api_calls import _file_id_to_url from ..exceptions import PyOpenMLError @@ -234,7 +235,7 @@ def publish(self): trace_arff = arff.dumps(self._generate_trace_arff_dict()) file_elements['trace'] = ("trace.arff", trace_arff) - return_value = _perform_api_call("/run/", file_elements=file_elements) + return_value = openml._api_calls._perform_api_call("/run/", file_elements=file_elements) run_id = int(xmltodict.parse(return_value)['oml:upload_run']['oml:run_id']) self.run_id = run_id return self @@ -370,7 +371,7 @@ def push_tag(self, tag): Tag to attach to the run. """ data = {'run_id': self.run_id, 'tag': tag} - _perform_api_call("/run/tag", data=data) + openml._api_calls._perform_api_call("/run/tag", data=data) def remove_tag(self, tag): """Removes a tag from this run on the server. @@ -381,7 +382,7 @@ def remove_tag(self, tag): Tag to attach to the run. """ data = {'run_id': self.run_id, 'tag': tag} - _perform_api_call("/run/untag", data=data) + openml._api_calls._perform_api_call("/run/untag", data=data) ################################################################################ diff --git a/openml/study/functions.py b/openml/study/functions.py index 535cf8dcd..cce4ca4b0 100644 --- a/openml/study/functions.py +++ b/openml/study/functions.py @@ -1,7 +1,7 @@ import xmltodict from openml.study import OpenMLStudy -from .._api_calls import _perform_api_call +import openml._api_calls def _multitag_to_list(result_dict, tag): @@ -22,7 +22,7 @@ def get_study(study_id, type=None): call_suffix = "study/%s" %str(study_id) if type is not None: call_suffix += "/" + type - xml_string = _perform_api_call(call_suffix) + xml_string = openml._api_calls._perform_api_call(call_suffix) result_dict = xmltodict.parse(xml_string)['oml:study'] id = int(result_dict['oml:id']) name = result_dict['oml:name'] @@ -56,4 +56,4 @@ def get_study(study_id, type=None): study = OpenMLStudy(id, name, description, creation_date, creator, tags, datasets, tasks, flows, setups) - return study \ No newline at end of file + return study diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index cf9e0a2a3..1a7864275 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -11,8 +11,8 @@ from ..datasets import get_dataset from .task import OpenMLTask, _create_task_cache_dir from .. import config -from .._api_calls import _perform_api_call import openml.utils +import openml._api_calls def _get_cached_tasks(): tasks = OrderedDict() @@ -60,7 +60,7 @@ def _get_estimation_procedure_list(): name, type, repeats, folds, stratified. """ - xml_string = _perform_api_call("estimationprocedure/list") + xml_string = openml._api_calls._perform_api_call("estimationprocedure/list") procs_dict = xmltodict.parse(xml_string) # Minimalistic check if the XML is useful if 'oml:estimationprocedures' not in procs_dict: @@ -175,7 +175,7 @@ def _list_tasks(task_type_id=None, **kwargs): def __list_tasks(api_call): - xml_string = _perform_api_call(api_call) + xml_string = openml._api_calls._perform_api_call(api_call) tasks_dict = xmltodict.parse(xml_string, force_list=('oml:task', 'oml:input')) # Minimalistic check if the XML is useful if 'oml:tasks' not in tasks_dict: @@ -301,7 +301,7 @@ def _get_task_description(task_id): return _get_cached_task(task_id) except OpenMLCacheException: xml_file = os.path.join(_create_task_cache_dir(task_id), "task.xml") - task_xml = _perform_api_call("task/%d" % task_id) + task_xml = openml._api_calls._perform_api_call("task/%d" % task_id) with io.open(xml_file, "w", encoding='utf8') as fh: fh.write(task_xml) diff --git a/openml/tasks/task.py b/openml/tasks/task.py index 98d2883f6..fb331b178 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -4,7 +4,8 @@ from .. import config from .. import datasets from .split import OpenMLSplit -from .._api_calls import _read_url, _perform_api_call +from .._api_calls import _read_url +import openml._api_calls class OpenMLTask(object): @@ -101,7 +102,7 @@ def push_tag(self, tag): Tag to attach to the task. """ data = {'task_id': self.task_id, 'tag': tag} - _perform_api_call("/task/tag", data=data) + openml._api_calls._perform_api_call("/task/tag", data=data) def remove_tag(self, tag): """Removes a tag from this task on the server. @@ -112,7 +113,7 @@ def remove_tag(self, tag): Tag to attach to the task. """ data = {'task_id': self.task_id, 'tag': tag} - _perform_api_call("/task/untag", data=data) + openml._api_calls._perform_api_call("/task/untag", data=data) def _create_task_cache_dir(task_id): diff --git a/openml/utils.py b/openml/utils.py index 1ea725957..1fe16ab04 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -1,6 +1,6 @@ import xmltodict import six -from ._api_calls import _perform_api_call +import openml._api_calls from openml.exceptions import OpenMLServerException @@ -79,7 +79,7 @@ def _tag_entity(entity_type, entity_id, tag, untag=False): post_variables = {'%s_id'%entity_type: entity_id, 'tag': tag} - result_xml = _perform_api_call(uri, post_variables) + result_xml = openml._api_calls._perform_api_call(uri, post_variables) result = xmltodict.parse(result_xml, force_list={'oml:tag'})[main_tag] @@ -158,4 +158,4 @@ def list_all(listing_call, *args, **filters): if limit < batch_size: batch_size = limit - return result \ No newline at end of file + return result diff --git a/tests/test_examples/test_OpenMLDemo.py b/tests/test_examples/test_OpenMLDemo.py index 168978945..bdadcdbb2 100644 --- a/tests/test_examples/test_OpenMLDemo.py +++ b/tests/test_examples/test_OpenMLDemo.py @@ -2,12 +2,24 @@ import shutil import sys +import matplotlib +matplotlib.use('AGG') import nbformat -from nbconvert.preprocessors import ExecutePreprocessor -from nbconvert.preprocessors.execute import CellExecutionError +from nbconvert.exporters import export +from nbconvert.exporters.python import PythonExporter +import six +if six.PY2: + import mock +else: + import unittest.mock as mock + +import openml._api_calls +import openml.config from openml.testing import TestBase +_perform_api_call = openml._api_calls._perform_api_call + class OpenMLDemoTest(TestBase): def setUp(self): @@ -29,29 +41,39 @@ def setUp(self): except: pass - def _test_notebook(self, notebook_name): + def _tst_notebook(self, notebook_name): notebook_filename = os.path.abspath(os.path.join( self.this_file_directory, '..', '..', 'examples', notebook_name)) - notebook_filename_out = os.path.join( - self.notebook_output_directory, notebook_name) with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) - nb.metadata.get('kernelspec', {})['name'] = self.kernel_name - ep = ExecutePreprocessor(kernel_name=self.kernel_name) - ep.timeout = 60 - - try: - ep.preprocess(nb, {'metadata': {'path': self.this_file_directory}}) - except CellExecutionError as e: - msg = 'Error executing the notebook "%s". ' % notebook_filename - msg += 'See notebook "%s" for the traceback.\n\n' % notebook_filename_out - msg += e.traceback - self.fail(msg) - finally: - with open(notebook_filename_out, mode='wt') as f: - nbformat.write(nb, f) - - def test_tutorial(self): - self._test_notebook('OpenML_Tutorial.ipynb') + + python_nb, metadata = export(PythonExporter, nb) + + # Remove magic lines manually + python_nb = '\n'.join([ + line for line in python_nb.split('\n') + if 'get_ipython().run_line_magic(' not in line + ]) + + exec(python_nb) + + @mock.patch('openml._api_calls._perform_api_call') + def test_tutorial(self, patch): + def side_effect(*args, **kwargs): + if ( + args[0].endswith('/run/') + and kwargs['file_elements'] is not None + ): + return """ + 1 + + """ + else: + return _perform_api_call(*args, **kwargs) + patch.side_effect = side_effect + + openml.config.server = self.production_server + self._tst_notebook('OpenML_Tutorial.ipynb') + self.assertGreater(patch.call_count, 100) diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index ca0f8ce13..54e3f28b1 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -197,7 +197,7 @@ def test_semi_legal_flow(self): flow.publish() @mock.patch('openml.flows.functions.get_flow') - @mock.patch('openml.flows.flow._perform_api_call') + @mock.patch('openml._api_calls._perform_api_call') def test_publish_error(self, api_call_mock, get_flow_mock): model = sklearn.ensemble.RandomForestClassifier() flow = openml.flows.sklearn_to_flow(model) From 2b2f8a2bac1bbd2146a5ede6aeb0cc06c6a267b6 Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Mon, 9 Apr 2018 16:06:51 +0200 Subject: [PATCH 80/86] Refactored setup.py and remove requirements.txt (#438) --- requirements.txt | 12 ------------ setup.py | 35 +++++++++++++++++++---------------- 2 files changed, 19 insertions(+), 28 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ad1e12cf0..000000000 --- a/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -mock -numpy>=1.6.2 -scipy>=0.13.3 -liac-arff>=2.2.1 -xmltodict -nose -requests -scikit-learn>=0.18 -nbconvert -nbformat -python-dateutil -oslo.concurrency diff --git a/setup.py b/setup.py index 0d4377209..a0cfb6e66 100644 --- a/setup.py +++ b/setup.py @@ -1,27 +1,12 @@ # -*- coding: utf-8 -*- -import os import setuptools import sys with open("openml/__version__.py") as fh: version = fh.readlines()[-1].split()[-1].strip("\"'") - -requirements_file = os.path.join(os.path.dirname(__file__), 'requirements.txt') -requirements = [] dependency_links = [] -with open(requirements_file) as fh: - for line in fh: - line = line.strip() - if line: - # Make sure the github URLs work here as well - split = line.split('@') - split = split[0] - split = split.split('/') - url = '/'.join(split[:-1]) - requirement = split[-1] - requirements.append(requirement) try: import numpy @@ -48,7 +33,25 @@ version=version, packages=setuptools.find_packages(), package_data={'': ['*.txt', '*.md']}, - install_requires=requirements, + install_requires=[ + 'mock', + 'numpy>=1.6.2', + 'scipy>=0.13.3', + 'liac-arff>=2.2.1', + 'xmltodict', + 'nose', + 'requests', + 'scikit-learn>=0.18', + 'nbformat', + 'python-dateutil', + 'oslo.concurrency', + ], + extras_require={ + 'test': [ + 'nbconvert', + 'jupyter_client' + ] + }, test_suite="nose.collector", classifiers=['Intended Audience :: Science/Research', 'Intended Audience :: Developers', From c626bdec64bfad950863878560ebe5f5a902da7c Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Tue, 10 Apr 2018 09:17:45 +0200 Subject: [PATCH 81/86] Solution which considers private datasets (#439) * Basic solution for private datasets * Faulty and not necessary implementation of finally * Fixed assertRaises call in test_get_data * Updated get_dataset * Fixed typo --- openml/datasets/functions.py | 22 +++++++++++++------ openml/exceptions.py | 8 ++++++- openml/tasks/functions.py | 2 +- tests/test_datasets/test_dataset_functions.py | 8 ++++++- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index fa6e235b0..48569ea81 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -4,6 +4,7 @@ import os import re import shutil +import six from oslo_concurrency import lockutils import xmltodict @@ -11,8 +12,8 @@ import openml.utils import openml._api_calls from .dataset import OpenMLDataset -from ..exceptions import OpenMLCacheException, OpenMLServerNoResult, \ - OpenMLHashException +from ..exceptions import OpenMLCacheException, OpenMLServerException, \ + OpenMLHashException, PrivateDatasetError from .. import config from .._api_calls import _read_url @@ -315,13 +316,21 @@ def get_dataset(dataset_id): did_cache_dir = _create_dataset_cache_directory(dataset_id) try: + remove_dataset_cache = True description = _get_dataset_description(did_cache_dir, dataset_id) arff_file = _get_dataset_arff(did_cache_dir, description) features = _get_dataset_features(did_cache_dir, dataset_id) qualities = _get_dataset_qualities(did_cache_dir, dataset_id) - except Exception as e: - _remove_dataset_cache_dir(did_cache_dir) - raise e + remove_dataset_cache = False + except OpenMLServerException as e: + # if there was an exception, check if the user had access to the dataset + if e.code == 112: + six.raise_from(PrivateDatasetError(e.message), None) + else: + raise e + finally: + if remove_dataset_cache: + _remove_dataset_cache_dir(did_cache_dir) dataset = _create_dataset_from_description( description, features, qualities, arff_file @@ -357,9 +366,8 @@ def _get_dataset_description(did_cache_dir, dataset_id): try: return _get_cached_dataset_description(dataset_id) - except (OpenMLCacheException): + except OpenMLCacheException: dataset_xml = openml._api_calls._perform_api_call("data/%d" % dataset_id) - with io.open(description_file, "w", encoding='utf8') as fh: fh.write(dataset_xml) diff --git a/openml/exceptions.py b/openml/exceptions.py index c162485d5..d38fdca91 100644 --- a/openml/exceptions.py +++ b/openml/exceptions.py @@ -43,4 +43,10 @@ def __init__(self, message): class OpenMLHashException(PyOpenMLError): """Locally computed hash is different than hash announced by the server.""" - pass \ No newline at end of file + pass + + +class PrivateDatasetError(PyOpenMLError): + "Exception thrown when the user has no rights to access the dataset" + def __init__(self, message): + super(PrivateDatasetError, self).__init__(message) \ No newline at end of file diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 1a7864275..512d86a2e 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -7,7 +7,7 @@ from oslo_concurrency import lockutils import xmltodict -from ..exceptions import OpenMLCacheException, OpenMLServerNoResult +from ..exceptions import OpenMLCacheException from ..datasets import get_dataset from .task import OpenMLTask, _create_task_cache_dir from .. import config diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 9469bcb1b..f208d4ea1 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -17,7 +17,8 @@ import openml from openml import OpenMLDataset -from openml.exceptions import OpenMLCacheException, PyOpenMLError, OpenMLHashException +from openml.exceptions import OpenMLCacheException, PyOpenMLError, \ + OpenMLHashException, PrivateDatasetError from openml.testing import TestBase from openml.utils import _tag_entity @@ -231,6 +232,11 @@ def test_get_dataset(self): self.assertGreater(len(dataset.features), 1) self.assertGreater(len(dataset.qualities), 4) + # Issue324 Properly handle private datasets when trying to access them + openml.config.server = self.production_server + self.assertRaises(PrivateDatasetError, openml.datasets.get_dataset, 45) + + def test_get_dataset_with_string(self): dataset = openml.datasets.get_dataset(101) self.assertRaises(PyOpenMLError, dataset._get_arff, 'arff') From 87ad7b1986825fa9bafa696172de30ec4eaeea90 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 11 Apr 2018 10:30:10 +0200 Subject: [PATCH 82/86] Have different cache directories for different servers (#432) * Have different directories for different servers * simplify based on jans comments * fix attribute access * Change variable name * Take into account Jans suggestions * Harmonize import, fix rebase errors * re-add accidentaly removed files * First try at a solution * Removing faulty fix * Fix for bug in unit test, method _get_cached_task * Removing FileNotFoundError as it does not exist in python2 * Fixing test_tagging * Changing id according to new solution * Change _remove_dataset_cache_dir to the new implementation --- doc/usage.rst | 2 +- openml/config.py | 79 ++++++--------- openml/datasets/functions.py | 95 ++++++------------- openml/runs/functions.py | 10 +- openml/runs/run.py | 5 +- openml/tasks/functions.py | 82 ++++------------ openml/tasks/split.py | 6 ++ openml/tasks/task.py | 10 +- openml/testing.py | 5 +- openml/utils.py | 76 ++++++++++++++- .../openml/test}/datasets/-1/dataset.arff | 0 .../openml/test}/datasets/-1/description.xml | 0 .../openml/test}/datasets/-1/features.xml | 0 .../openml/test}/datasets/-1/qualities.xml | 0 .../openml/test}/datasets/2/dataset.arff | 0 .../openml/test}/datasets/2/description.xml | 0 .../openml/test}/datasets/2/features.xml | 0 .../openml/test}/datasets/2/qualities.xml | 0 .../openml/test}/runs/1/description.xml | 0 .../openml/test}/setups/1/description.xml | 0 .../openml/test}/tasks/1/datasplits.arff | 0 .../{ => org/openml/test}/tasks/1/task.xml | 0 .../openml/test}/tasks/1882/datasplits.arff | 0 .../{ => org/openml/test}/tasks/1882/task.xml | 0 .../openml/test}/tasks/3/datasplits.arff | 0 .../{ => org/openml/test}/tasks/3/task.xml | 0 tests/test_datasets/test_dataset_functions.py | 37 ++++---- .../test_evaluation_functions.py | 2 +- tests/test_runs/test_run.py | 7 +- tests/test_runs/test_run_functions.py | 6 +- tests/test_setups/test_setup_functions.py | 6 +- tests/test_study/test_study_functions.py | 2 +- tests/test_tasks/test_split.py | 4 +- tests/test_tasks/test_task.py | 2 +- tests/test_tasks/test_task_functions.py | 37 ++++---- 35 files changed, 240 insertions(+), 233 deletions(-) rename tests/files/{ => org/openml/test}/datasets/-1/dataset.arff (100%) rename tests/files/{ => org/openml/test}/datasets/-1/description.xml (100%) rename tests/files/{ => org/openml/test}/datasets/-1/features.xml (100%) rename tests/files/{ => org/openml/test}/datasets/-1/qualities.xml (100%) rename tests/files/{ => org/openml/test}/datasets/2/dataset.arff (100%) rename tests/files/{ => org/openml/test}/datasets/2/description.xml (100%) rename tests/files/{ => org/openml/test}/datasets/2/features.xml (100%) rename tests/files/{ => org/openml/test}/datasets/2/qualities.xml (100%) rename tests/files/{ => org/openml/test}/runs/1/description.xml (100%) rename tests/files/{ => org/openml/test}/setups/1/description.xml (100%) rename tests/files/{ => org/openml/test}/tasks/1/datasplits.arff (100%) rename tests/files/{ => org/openml/test}/tasks/1/task.xml (100%) rename tests/files/{ => org/openml/test}/tasks/1882/datasplits.arff (100%) rename tests/files/{ => org/openml/test}/tasks/1882/task.xml (100%) rename tests/files/{ => org/openml/test}/tasks/3/datasplits.arff (100%) rename tests/files/{ => org/openml/test}/tasks/3/task.xml (100%) diff --git a/doc/usage.rst b/doc/usage.rst index 0801c2c03..a4bf8ee0b 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -55,7 +55,7 @@ API: .. code:: python >>> import os - >>> openml.config.set_cache_directory(os.path.expanduser('~/.openml/cache')) + >>> openml.config.cache_directory = os.path.expanduser('~/.openml/cache') Config file: diff --git a/openml/config.py b/openml/config.py index 192b5fcaa..949fe869f 100644 --- a/openml/config.py +++ b/openml/config.py @@ -6,6 +6,7 @@ from six import StringIO from six.moves import configparser +from six.moves.urllib_parse import urlparse logger = logging.getLogger(__name__) @@ -13,10 +14,23 @@ format='[%(levelname)s] [%(asctime)s:%(name)s] %(' 'message)s', datefmt='%H:%M:%S') +# Default values! +_defaults = { + 'apikey': None, + 'server': "https://www.openml.org/api/v1/xml", + 'verbosity': 0, + 'cachedir': os.path.expanduser('~/.openml/cache'), + 'avoid_duplicate_runs': 'True', +} + config_file = os.path.expanduser('~/.openml/config') -server = "https://www.openml.org/api/v1/xml" + +# Default values are actually added here in the _setup() function which is +# called at the end of this module +server = "" apikey = "" -cachedir = "" +# The current cache directory (without the server name) +cache_directory = "" def _setup(): @@ -26,12 +40,11 @@ def _setup(): key and server can be set by the user simply using openml.config.apikey = THEIRKEY openml.config.server = SOMESERVER - The cache dir needs to be set up calling set_cache_directory - because it needs some setup. We could also make it a property but that's less clear. """ global apikey global server + global cache_directory global avoid_duplicate_runs # read config file, create cache directory try: @@ -42,52 +55,15 @@ def _setup(): config = _parse_config() apikey = config.get('FAKE_SECTION', 'apikey') server = config.get('FAKE_SECTION', 'server') - cache_dir = config.get('FAKE_SECTION', 'cachedir') + cache_directory = config.get('FAKE_SECTION', 'cachedir') avoid_duplicate_runs = config.getboolean('FAKE_SECTION', 'avoid_duplicate_runs') - set_cache_directory(cache_dir) - - -def set_cache_directory(cachedir): - """Set module-wide cache directory. - - Sets the cache directory into which to download datasets, tasks etc. - - Parameters - ---------- - cachedir : string - Path to use as cache directory. - - See also - -------- - get_cache_directory - """ - - global _cachedir - _cachedir = cachedir - - # Set up the cache directories - dataset_cache_dir = os.path.join(cachedir, "datasets") - task_cache_dir = os.path.join(cachedir, "tasks") - run_cache_dir = os.path.join(cachedir, 'runs') - lock_dir = os.path.join(cachedir, 'locks') - - for dir_ in [ - cachedir, dataset_cache_dir, task_cache_dir, run_cache_dir, lock_dir, - ]: - if not os.path.exists(dir_) and not os.path.isdir(dir_): - os.mkdir(dir_) def _parse_config(): """Parse the config file, set up defaults. """ - defaults = {'apikey': apikey, - 'server': server, - 'verbosity': 0, - 'cachedir': os.path.expanduser('~/.openml/cache'), - 'avoid_duplicate_runs': 'True'} - config = configparser.RawConfigParser(defaults=defaults) + config = configparser.RawConfigParser(defaults=_defaults) if not os.path.exists(config_file): # Create an empty config file if there was none so far @@ -106,8 +82,7 @@ def _parse_config(): config_file_.seek(0) config.readfp(config_file_) except OSError as e: - logging.info("Error opening file %s: %s" % - config_file, e.message) + logging.info("Error opening file %s: %s", config_file, e.message) return config @@ -119,13 +94,19 @@ def get_cache_directory(): cachedir : string The current cache directory. - See also - -------- - set_cache_directory """ + url_suffix = urlparse(server).netloc + reversed_url_suffix = '/'.join(url_suffix.split('.')[::-1]) + if not cache_directory: + _cachedir = _defaults(cache_directory) + else: + _cachedir = cache_directory + _cachedir = os.path.join(_cachedir, reversed_url_suffix) return _cachedir -__all__ = ["set_cache_directory", 'get_cache_directory'] +__all__ = [ + 'get_cache_directory', +] _setup() diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 48569ea81..b447c671d 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -14,13 +14,22 @@ from .dataset import OpenMLDataset from ..exceptions import OpenMLCacheException, OpenMLServerException, \ OpenMLHashException, PrivateDatasetError -from .. import config -from .._api_calls import _read_url +from ..utils import ( + _create_cache_directory, + _remove_cache_dir_for_id, + _create_cache_directory_for_id, + _create_lockfiles_dir, +) + + +DATASETS_CACHE_DIR_NAME = 'datasets' + ############################################################################ # Local getters/accessors to the cache directory + def _list_cached_datasets(): """Return list with ids of all cached datasets @@ -31,8 +40,7 @@ def _list_cached_datasets(): """ datasets = [] - dataset_cache = config.get_cache_directory() - dataset_cache_dir = os.path.join(dataset_cache, "datasets") + dataset_cache_dir = _create_cache_directory(DATASETS_CACHE_DIR_NAME) directory_content = os.listdir(dataset_cache_dir) directory_content.sort() @@ -88,8 +96,9 @@ def _get_cached_dataset(dataset_id): def _get_cached_dataset_description(dataset_id): - cache_dir = config.get_cache_directory() - did_cache_dir = os.path.join(cache_dir, "datasets", str(dataset_id)) + did_cache_dir = _create_cache_directory_for_id( + DATASETS_CACHE_DIR_NAME, dataset_id, + ) description_file = os.path.join(did_cache_dir, "description.xml") try: with io.open(description_file, encoding='utf8') as fh: @@ -102,8 +111,9 @@ def _get_cached_dataset_description(dataset_id): def _get_cached_dataset_features(dataset_id): - cache_dir = config.get_cache_directory() - did_cache_dir = os.path.join(cache_dir, "datasets", str(dataset_id)) + did_cache_dir = _create_cache_directory_for_id( + DATASETS_CACHE_DIR_NAME, dataset_id, + ) features_file = os.path.join(did_cache_dir, "features.xml") try: with io.open(features_file, encoding='utf8') as fh: @@ -115,8 +125,9 @@ def _get_cached_dataset_features(dataset_id): def _get_cached_dataset_qualities(dataset_id): - cache_dir = config.get_cache_directory() - did_cache_dir = os.path.join(cache_dir, "datasets", str(dataset_id)) + did_cache_dir = _create_cache_directory_for_id( + DATASETS_CACHE_DIR_NAME, dataset_id, + ) qualities_file = os.path.join(did_cache_dir, "qualities.xml") try: with io.open(qualities_file, encoding='utf8') as fh: @@ -128,8 +139,9 @@ def _get_cached_dataset_qualities(dataset_id): def _get_cached_dataset_arff(dataset_id): - cache_dir = config.get_cache_directory() - did_cache_dir = os.path.join(cache_dir, "datasets", str(dataset_id)) + did_cache_dir = _create_cache_directory_for_id( + DATASETS_CACHE_DIR_NAME, dataset_id, + ) output_file = os.path.join(did_cache_dir, "dataset.arff") try: @@ -311,9 +323,11 @@ def get_dataset(dataset_id): with lockutils.external_lock( name='datasets.functions.get_dataset:%d' % dataset_id, - lock_path=os.path.join(config.get_cache_directory(), 'locks'), + lock_path=_create_lockfiles_dir(), ): - did_cache_dir = _create_dataset_cache_directory(dataset_id) + did_cache_dir = _create_cache_directory_for_id( + DATASETS_CACHE_DIR_NAME, dataset_id, + ) try: remove_dataset_cache = True @@ -330,7 +344,7 @@ def get_dataset(dataset_id): raise e finally: if remove_dataset_cache: - _remove_dataset_cache_dir(did_cache_dir) + _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir) dataset = _create_dataset_from_description( description, features, qualities, arff_file @@ -412,7 +426,7 @@ def _get_dataset_arff(did_cache_dir, description): pass url = description['oml:url'] - arff_string = _read_url(url) + arff_string = openml._api_calls._read_url(url) md5 = hashlib.md5() md5.update(arff_string.encode('utf-8')) md5_checksum = md5.hexdigest() @@ -505,55 +519,6 @@ def _get_dataset_qualities(did_cache_dir, dataset_id): return qualities -def _create_dataset_cache_directory(dataset_id): - """Create a dataset cache directory - - In order to have a clearer cache structure and because every dataset - is cached in several files (description, arff, features, qualities), there - is a directory for each dataset witch the dataset ID being the directory - name. This function creates this cache directory. - - This function is NOT thread/multiprocessing safe. - - Parameters - ---------- - did : int - Dataset ID - - Returns - ------- - str - Path of the created dataset cache directory. - """ - dataset_cache_dir = os.path.join( - config.get_cache_directory(), - "datasets", - str(dataset_id), - ) - if os.path.exists(dataset_cache_dir) and os.path.isdir(dataset_cache_dir): - pass - elif os.path.exists(dataset_cache_dir) and not os.path.isdir(dataset_cache_dir): - raise ValueError('Dataset cache dir exists but is not a directory!') - else: - os.makedirs(dataset_cache_dir) - return dataset_cache_dir - - -def _remove_dataset_cache_dir(did_cache_dir): - """Remove the dataset cache directory - - This function is NOT thread/multiprocessing safe. - - Parameters - ---------- - """ - try: - shutil.rmtree(did_cache_dir) - except (OSError, IOError): - raise ValueError('Cannot remove faulty dataset cache directory %s.' - 'Please do this manually!' % did_cache_dir) - - def _create_dataset_from_description(description, features, qualities, arff_file): """Create a dataset object from a description dict. diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 5190797c7..e12c4ccd7 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -2,6 +2,7 @@ import io import json import os +import shutil import sys import time import warnings @@ -28,6 +29,8 @@ # _get_version_info, _get_dict and _create_setup_string are in run.py to avoid # circular imports +RUNS_CACHE_DIR_NAME = 'runs' + def run_model_on_task(task, model, avoid_duplicate_runs=True, flow_tags=None, seed=None): @@ -643,7 +646,7 @@ def get_run(run_id): run : OpenMLRun Run corresponding to ID, fetched from the server. """ - run_dir = os.path.join(config.get_cache_directory(), "runs", str(run_id)) + run_dir = openml.utils._create_cache_directory_for_id(RUNS_CACHE_DIR_NAME, run_id) run_file = os.path.join(run_dir, "description.xml") if not os.path.exists(run_dir): @@ -878,8 +881,9 @@ def _create_trace_from_arff(arff_obj): def _get_cached_run(run_id): """Load a run from the cache.""" - cache_dir = config.get_cache_directory() - run_cache_dir = os.path.join(cache_dir, "runs", str(run_id)) + run_cache_dir = openml.utils._create_cache_directory_for_id( + RUNS_CACHE_DIR_NAME, run_id, + ) try: run_file = os.path.join(run_cache_dir, "description.xml") with io.open(run_file, encoding='utf8') as fh: diff --git a/openml/runs/run.py b/openml/runs/run.py index 7a01433c5..9d80999d6 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -10,7 +10,6 @@ import openml import openml._api_calls from ..tasks import get_task -from .._api_calls import _file_id_to_url from ..exceptions import PyOpenMLError @@ -142,7 +141,9 @@ def get_metric_fn(self, sklearn_fn, kwargs={}): if self.data_content is not None and self.task_id is not None: predictions_arff = self._generate_arff_dict() elif 'predictions' in self.output_files: - predictions_file_url = _file_id_to_url(self.output_files['predictions'], 'predictions.arff') + predictions_file_url = openml._api_calls._file_id_to_url( + self.output_files['predictions'], 'predictions.arff', + ) predictions_arff = arff.loads(openml._api_calls._read_url(predictions_file_url)) # TODO: make this a stream reader else: diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 512d86a2e..0fbdc9b21 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -2,26 +2,25 @@ import io import re import os -import shutil from oslo_concurrency import lockutils import xmltodict from ..exceptions import OpenMLCacheException from ..datasets import get_dataset -from .task import OpenMLTask, _create_task_cache_dir -from .. import config +from .task import OpenMLTask import openml.utils import openml._api_calls +TASKS_CACHE_DIR_NAME = 'tasks' + + def _get_cached_tasks(): tasks = OrderedDict() - cache_dir = config.get_cache_directory() - task_cache_dir = os.path.join(cache_dir, "tasks") + task_cache_dir = openml.utils._create_cache_directory(TASKS_CACHE_DIR_NAME) directory_content = os.listdir(task_cache_dir) directory_content.sort() - # Find all dataset ids for which we have downloaded the dataset # description @@ -36,15 +35,19 @@ def _get_cached_tasks(): def _get_cached_task(tid): - cache_dir = config.get_cache_directory() - task_cache_dir = os.path.join(cache_dir, "tasks") - task_file = os.path.join(task_cache_dir, str(tid), "task.xml") + + tid_cache_dir = openml.utils._create_cache_directory_for_id( + TASKS_CACHE_DIR_NAME, + tid + ) + task_file = os.path.join(tid_cache_dir, "task.xml") try: with io.open(task_file, encoding='utf8') as fh: task = _create_task_from_xml(xml=fh.read()) return task except (OSError, IOError): + openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir) raise OpenMLCacheException("Task file for tid %d not " "cached" % tid) @@ -275,11 +278,13 @@ def get_task(task_id): raise ValueError("Task ID is neither an Integer nor can be " "cast to an Integer.") - tid_cache_dir = _create_task_cache_dir(task_id) + tid_cache_dir = openml.utils._create_cache_directory_for_id( + TASKS_CACHE_DIR_NAME, task_id, + ) with lockutils.external_lock( name='task.functions.get_task:%d' % task_id, - lock_path=os.path.join(config.get_cache_directory(), 'locks'), + lock_path=openml.utils._create_lockfiles_dir(), ): try: task = _get_task_description(task_id) @@ -287,9 +292,8 @@ def get_task(task_id): class_labels = dataset.retrieve_class_labels(task.target_name) task.class_labels = class_labels task.download_split() - except Exception as e: - _remove_task_cache_dir(tid_cache_dir) + openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir) raise e return task @@ -300,7 +304,10 @@ def _get_task_description(task_id): try: return _get_cached_task(task_id) except OpenMLCacheException: - xml_file = os.path.join(_create_task_cache_dir(task_id), "task.xml") + xml_file = os.path.join( + openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id), + "task.xml", + ) task_xml = openml._api_calls._perform_api_call("task/%d" % task_id) with io.open(xml_file, "w", encoding='utf8') as fh: @@ -310,53 +317,6 @@ def _get_task_description(task_id): return task -def _create_task_cache_directory(task_id): - """Create a task cache directory - - In order to have a clearer cache structure and because every task - is cached in several files (description, split), there - is a directory for each task witch the task ID being the directory - name. This function creates this cache directory. - - This function is NOT thread/multiprocessing safe. - - Parameters - ---------- - tid : int - Task ID - - Returns - ------- - str - Path of the created dataset cache directory. - """ - task_cache_dir = os.path.join( - config.get_cache_directory(), "tasks", str(task_id) - ) - if os.path.exists(task_cache_dir) and os.path.isdir(task_cache_dir): - pass - elif os.path.exists(task_cache_dir) and not os.path.isdir(task_cache_dir): - raise ValueError('Task cache dir exists but is not a directory!') - else: - os.makedirs(task_cache_dir) - return task_cache_dir - - -def _remove_task_cache_dir(tid_cache_dir): - """Remove the task cache directory - - This function is NOT thread/multiprocessing safe. - - Parameters - ---------- - """ - try: - shutil.rmtree(tid_cache_dir) - except (OSError, IOError): - raise ValueError('Cannot remove faulty task cache directory %s.' - 'Please do this manually!' % tid_cache_dir) - - def _create_task_from_xml(xml): dic = xmltodict.parse(xml)["oml:task"] diff --git a/openml/tasks/split.py b/openml/tasks/split.py index ae7f3a85f..6f4b13730 100644 --- a/openml/tasks/split.py +++ b/openml/tasks/split.py @@ -10,6 +10,10 @@ Split = namedtuple("Split", ["train", "test"]) +if six.PY2: + FileNotFoundError = IOError + + class OpenMLSplit(object): def __init__(self, name, description, split): @@ -78,6 +82,8 @@ def _from_arff_file(cls, filename, cache=True): # Cache miss if repetitions is None: # Faster than liac-arff and sufficient in this situation! + if not os.path.exists(filename): + raise FileNotFoundError('Split arff %s does not exist!' % filename) splits, meta = scipy.io.arff.loadarff(filename) name = meta.name diff --git a/openml/tasks/task.py b/openml/tasks/task.py index fb331b178..cc7dd6731 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -4,8 +4,8 @@ from .. import config from .. import datasets from .split import OpenMLSplit -from .._api_calls import _read_url import openml._api_calls +from ..utils import _create_cache_directory_for_id class OpenMLTask(object): @@ -64,7 +64,7 @@ def _download_split(self, cache_file): pass except (OSError, IOError): split_url = self.estimation_procedure["data_splits_url"] - split_arff = _read_url(split_url) + split_arff = openml._api_calls._read_url(split_url) with io.open(cache_file, "w", encoding='utf8') as fh: fh.write(split_arff) @@ -74,12 +74,12 @@ def download_split(self): """Download the OpenML split for a given task. """ cached_split_file = os.path.join( - _create_task_cache_dir(self.task_id), "datasplits.arff") + _create_cache_directory_for_id('tasks', self.task_id), + "datasplits.arff", + ) try: split = OpenMLSplit._from_arff_file(cached_split_file) - # Add FileNotFoundError in python3 version (which should be a - # subclass of OSError. except (OSError, IOError): # Next, download and cache the associated split file self._download_split(cached_split_file) diff --git a/openml/testing.py b/openml/testing.py index 62c383a95..0b75da06f 100644 --- a/openml/testing.py +++ b/openml/testing.py @@ -26,7 +26,6 @@ def setUp(self): self.maxDiff = None self.static_cache_dir = None static_cache_dir = os.path.dirname(os.path.abspath(inspect.getfile(self.__class__))) - static_cache_dir = os.path.abspath(os.path.join(static_cache_dir, '..')) content = os.listdir(static_cache_dir) if 'files' in content: @@ -52,10 +51,12 @@ def setUp(self): openml.config.apikey = "610344db6388d9ba34f6db45a3cf71de" self.production_server = openml.config.server self.test_server = "https://test.openml.org/api/v1/xml" + openml.config.cache_directory = None + openml.config.server = self.test_server openml.config.avoid_duplicate_runs = False - openml.config.set_cache_directory(self.workdir) + openml.config.cache_directory = self.workdir # If we're on travis, we save the api key in the config file to allow # the notebook tests to read them. diff --git a/openml/utils.py b/openml/utils.py index 1fe16ab04..afe83f141 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -1,9 +1,13 @@ +import os import xmltodict import six -import openml._api_calls +import shutil +import openml._api_calls +from . import config from openml.exceptions import OpenMLServerException + def extract_xml_tags(xml_tag_name, node, allow_none=True): """Helper to extract xml tags from xmltodict. @@ -159,3 +163,73 @@ def list_all(listing_call, *args, **filters): batch_size = limit return result + + +def _create_cache_directory(key): + cache = config.get_cache_directory() + cache_dir = os.path.join(cache, key) + try: + os.makedirs(cache_dir) + except: + pass + return cache_dir + + +def _create_cache_directory_for_id(key, id_): + """Create the cache directory for a specific ID + + In order to have a clearer cache structure and because every task + is cached in several files (description, split), there + is a directory for each task witch the task ID being the directory + name. This function creates this cache directory. + + This function is NOT thread/multiprocessing safe. + + Parameters + ---------- + key : str + + id_ : int + + Returns + ------- + str + Path of the created dataset cache directory. + """ + cache_dir = os.path.join( + _create_cache_directory(key), str(id_) + ) + if os.path.exists(cache_dir) and os.path.isdir(cache_dir): + pass + elif os.path.exists(cache_dir) and not os.path.isdir(cache_dir): + raise ValueError('%s cache dir exists but is not a directory!' % key) + else: + os.makedirs(cache_dir) + return cache_dir + + +def _remove_cache_dir_for_id(key, cache_dir): + """Remove the task cache directory + + This function is NOT thread/multiprocessing safe. + + Parameters + ---------- + key : str + + cache_dir : str + """ + try: + shutil.rmtree(cache_dir) + except (OSError, IOError): + raise ValueError('Cannot remove faulty %s cache directory %s.' + 'Please do this manually!' % (key, cache_dir)) + + +def _create_lockfiles_dir(): + dir = os.path.join(config.get_cache_directory(), 'locks') + try: + os.makedirs(dir) + except: + pass + return dir diff --git a/tests/files/datasets/-1/dataset.arff b/tests/files/org/openml/test/datasets/-1/dataset.arff similarity index 100% rename from tests/files/datasets/-1/dataset.arff rename to tests/files/org/openml/test/datasets/-1/dataset.arff diff --git a/tests/files/datasets/-1/description.xml b/tests/files/org/openml/test/datasets/-1/description.xml similarity index 100% rename from tests/files/datasets/-1/description.xml rename to tests/files/org/openml/test/datasets/-1/description.xml diff --git a/tests/files/datasets/-1/features.xml b/tests/files/org/openml/test/datasets/-1/features.xml similarity index 100% rename from tests/files/datasets/-1/features.xml rename to tests/files/org/openml/test/datasets/-1/features.xml diff --git a/tests/files/datasets/-1/qualities.xml b/tests/files/org/openml/test/datasets/-1/qualities.xml similarity index 100% rename from tests/files/datasets/-1/qualities.xml rename to tests/files/org/openml/test/datasets/-1/qualities.xml diff --git a/tests/files/datasets/2/dataset.arff b/tests/files/org/openml/test/datasets/2/dataset.arff similarity index 100% rename from tests/files/datasets/2/dataset.arff rename to tests/files/org/openml/test/datasets/2/dataset.arff diff --git a/tests/files/datasets/2/description.xml b/tests/files/org/openml/test/datasets/2/description.xml similarity index 100% rename from tests/files/datasets/2/description.xml rename to tests/files/org/openml/test/datasets/2/description.xml diff --git a/tests/files/datasets/2/features.xml b/tests/files/org/openml/test/datasets/2/features.xml similarity index 100% rename from tests/files/datasets/2/features.xml rename to tests/files/org/openml/test/datasets/2/features.xml diff --git a/tests/files/datasets/2/qualities.xml b/tests/files/org/openml/test/datasets/2/qualities.xml similarity index 100% rename from tests/files/datasets/2/qualities.xml rename to tests/files/org/openml/test/datasets/2/qualities.xml diff --git a/tests/files/runs/1/description.xml b/tests/files/org/openml/test/runs/1/description.xml similarity index 100% rename from tests/files/runs/1/description.xml rename to tests/files/org/openml/test/runs/1/description.xml diff --git a/tests/files/setups/1/description.xml b/tests/files/org/openml/test/setups/1/description.xml similarity index 100% rename from tests/files/setups/1/description.xml rename to tests/files/org/openml/test/setups/1/description.xml diff --git a/tests/files/tasks/1/datasplits.arff b/tests/files/org/openml/test/tasks/1/datasplits.arff similarity index 100% rename from tests/files/tasks/1/datasplits.arff rename to tests/files/org/openml/test/tasks/1/datasplits.arff diff --git a/tests/files/tasks/1/task.xml b/tests/files/org/openml/test/tasks/1/task.xml similarity index 100% rename from tests/files/tasks/1/task.xml rename to tests/files/org/openml/test/tasks/1/task.xml diff --git a/tests/files/tasks/1882/datasplits.arff b/tests/files/org/openml/test/tasks/1882/datasplits.arff similarity index 100% rename from tests/files/tasks/1882/datasplits.arff rename to tests/files/org/openml/test/tasks/1882/datasplits.arff diff --git a/tests/files/tasks/1882/task.xml b/tests/files/org/openml/test/tasks/1882/task.xml similarity index 100% rename from tests/files/tasks/1882/task.xml rename to tests/files/org/openml/test/tasks/1882/task.xml diff --git a/tests/files/tasks/3/datasplits.arff b/tests/files/org/openml/test/tasks/3/datasplits.arff similarity index 100% rename from tests/files/tasks/3/datasplits.arff rename to tests/files/org/openml/test/tasks/3/datasplits.arff diff --git a/tests/files/tasks/3/task.xml b/tests/files/org/openml/test/tasks/3/task.xml similarity index 100% rename from tests/files/tasks/3/task.xml rename to tests/files/org/openml/test/tasks/3/task.xml diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index f208d4ea1..24c2bb77c 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -20,7 +20,7 @@ from openml.exceptions import OpenMLCacheException, PyOpenMLError, \ OpenMLHashException, PrivateDatasetError from openml.testing import TestBase -from openml.utils import _tag_entity +from openml.utils import _tag_entity, _create_cache_directory_for_id from openml.datasets.functions import (_get_cached_dataset, _get_cached_dataset_features, @@ -29,7 +29,8 @@ _get_dataset_description, _get_dataset_arff, _get_dataset_features, - _get_dataset_qualities) + _get_dataset_qualities, + DATASETS_CACHE_DIR_NAME) class TestOpenMLDataset(TestBase): @@ -57,7 +58,7 @@ def _remove_pickle_files(self): pass def test__list_cached_datasets(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir cached_datasets = openml.datasets.functions._list_cached_datasets() self.assertIsInstance(cached_datasets, list) self.assertEqual(len(cached_datasets), 2) @@ -65,7 +66,7 @@ def test__list_cached_datasets(self): @mock.patch('openml.datasets.functions._list_cached_datasets') def test__get_cached_datasets(self, _list_cached_datasets_mock): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir _list_cached_datasets_mock.return_value = [-1, 2] datasets = _get_cached_datasets() self.assertIsInstance(datasets, dict) @@ -73,7 +74,7 @@ def test__get_cached_datasets(self, _list_cached_datasets_mock): self.assertIsInstance(list(datasets.values())[0], OpenMLDataset) def test__get_cached_dataset(self, ): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir dataset = _get_cached_dataset(2) features = _get_cached_dataset_features(2) qualities = _get_cached_dataset_qualities(2) @@ -83,25 +84,25 @@ def test__get_cached_dataset(self, ): self.assertTrue(len(dataset.qualities) == len(qualities)) def test_get_cached_dataset_description(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir description = openml.datasets.functions._get_cached_dataset_description(2) self.assertIsInstance(description, dict) def test_get_cached_dataset_description_not_cached(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir self.assertRaisesRegexp(OpenMLCacheException, "Dataset description for " "dataset id 3 not cached", openml.datasets.functions._get_cached_dataset_description, 3) def test_get_cached_dataset_arff(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir description = openml.datasets.functions._get_cached_dataset_arff( dataset_id=2) self.assertIsInstance(description, str) def test_get_cached_dataset_arff_not_cached(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir self.assertRaisesRegexp(OpenMLCacheException, "ARFF file for " "dataset id 3 not cached", openml.datasets.functions._get_cached_dataset_arff, @@ -185,7 +186,6 @@ def test_list_datasets_empty(self): self.assertIsInstance(datasets, dict) - @unittest.skip('See https://github.com/openml/openml-python/issues/149') def test_check_datasets_active(self): active = openml.datasets.check_datasets_active([1, 17]) @@ -261,7 +261,7 @@ def test__get_dataset_description(self): self.assertTrue(os.path.exists(description_xml_path)) def test__getarff_path_dataset_arff(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir description = openml.datasets.functions._get_cached_dataset_description(2) arff_path = _get_dataset_arff(self.workdir, description) self.assertIsInstance(arff_path, str) @@ -294,10 +294,13 @@ def test__get_dataset_qualities(self): def test_deletion_of_cache_dir(self): # Simple removal - did_cache_dir = openml.datasets.functions.\ - _create_dataset_cache_directory(1) + did_cache_dir = openml.utils._create_cache_directory_for_id( + DATASETS_CACHE_DIR_NAME, 1, + ) self.assertTrue(os.path.exists(did_cache_dir)) - openml.datasets.functions._remove_dataset_cache_dir(did_cache_dir) + openml.utils._remove_cache_dir_for_id( + DATASETS_CACHE_DIR_NAME, did_cache_dir, + ) self.assertFalse(os.path.exists(did_cache_dir)) # Use _get_dataset_arff to load the description, trigger an exception in the @@ -307,7 +310,9 @@ def test_deletion_of_cache_dir_faulty_download(self, patch): patch.side_effect = Exception('Boom!') self.assertRaisesRegexp(Exception, 'Boom!', openml.datasets.get_dataset, 1) - datasets_cache_dir = os.path.join(self.workdir, 'datasets') + datasets_cache_dir = os.path.join( + self.workdir, 'org', 'openml', 'test', 'datasets' + ) self.assertEqual(len(os.listdir(datasets_cache_dir)), 0) def test_publish_dataset(self): @@ -321,7 +326,7 @@ def test_publish_dataset(self): self.assertIsInstance(dataset.dataset_id, int) def test__retrieve_class_labels(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir labels = openml.datasets.get_dataset(2).retrieve_class_labels() self.assertEqual(labels, ['1', '2', '3', '4', '5', 'U']) labels = openml.datasets.get_dataset(2).retrieve_class_labels( diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 771ee2cd4..be55c2cd8 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -69,4 +69,4 @@ def test_list_evaluations_empty(self): if len(evaluations) > 0: raise ValueError('UnitTest Outdated, got somehow results') - self.assertIsInstance(evaluations, dict) \ No newline at end of file + self.assertIsInstance(evaluations, dict) diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index eccda841d..deafbcacc 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -52,14 +52,17 @@ def test_parse_parameters(self): self.assertEqual(parameter['oml:component'], 2) def test_tagging(self): - run = openml.runs.get_run(1) + + runs = openml.runs.list_runs(size=1) + run_id = list(runs.keys())[0] + run = openml.runs.get_run(run_id) tag = "testing_tag_{}_{}".format(self.id(), time()) run_list = openml.runs.list_runs(tag=tag) self.assertEqual(len(run_list), 0) run.push_tag(tag) run_list = openml.runs.list_runs(tag=tag) self.assertEqual(len(run_list), 1) - self.assertIn(1, run_list) + self.assertIn(run_id, run_list) run.remove_tag(tag) run_list = openml.runs.list_runs(tag=tag) self.assertEqual(len(run_list), 0) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index d28a834b3..f824e1ed1 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -987,10 +987,10 @@ def test_predict_proba_hardclassifier(self): np.testing.assert_array_equal(predictionsA, predictionsB) def test_get_cached_run(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir openml.runs.functions._get_cached_run(1) def test_get_uncached_run(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir with self.assertRaises(openml.exceptions.OpenMLCacheException): - openml.runs.functions._get_cached_run(10) \ No newline at end of file + openml.runs.functions._get_cached_run(10) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index e2c705a6e..928874837 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -159,11 +159,11 @@ def test_setuplist_offset(self): self.assertEqual(len(all), size * 2) def test_get_cached_setup(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir openml.setups.functions._get_cached_setup(1) def test_get_uncached_setup(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir with self.assertRaises(openml.exceptions.OpenMLCacheException): - openml.setups.functions._get_cached_setup(10) \ No newline at end of file + openml.setups.functions._get_cached_setup(10) diff --git a/tests/test_study/test_study_functions.py b/tests/test_study/test_study_functions.py index 0bf0496da..c2d0b7258 100644 --- a/tests/test_study/test_study_functions.py +++ b/tests/test_study/test_study_functions.py @@ -23,4 +23,4 @@ def test_get_tasks(self): self.assertEquals(study.data, None) self.assertGreater(len(study.tasks), 0) self.assertEquals(study.flows, None) - self.assertEquals(study.setups, None) \ No newline at end of file + self.assertEquals(study.setups, None) diff --git a/tests/test_tasks/test_split.py b/tests/test_tasks/test_split.py index e58e2dc2d..6fd2926e5 100644 --- a/tests/test_tasks/test_split.py +++ b/tests/test_tasks/test_split.py @@ -16,7 +16,9 @@ def setUp(self): self.directory = os.path.dirname(__file__) # This is for dataset self.arff_filename = os.path.join( - self.directory, "..", "files", "tasks", "1882", "datasplits.arff") + self.directory, "..", "files", "org", "openml", "test", + "tasks", "1882", "datasplits.arff" + ) self.pd_filename = self.arff_filename.replace(".arff", ".pkl") def tearDown(self): diff --git a/tests/test_tasks/test_task.py b/tests/test_tasks/test_task.py index 704ce8f39..fdbfa06d1 100644 --- a/tests/test_tasks/test_task.py +++ b/tests/test_tasks/test_task.py @@ -59,7 +59,7 @@ def test_tagging(self): self.assertEqual(len(task_list), 0) def test_get_train_and_test_split_indices(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir task = openml.tasks.get_task(1882) train_indices, test_indices = task.get_train_test_split_indices(0, 0) self.assertEqual(16, train_indices[0]) diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index b9d4368e7..a711534c6 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -18,19 +18,19 @@ class TestTask(TestBase): _multiprocess_can_split_ = True def test__get_cached_tasks(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir tasks = openml.tasks.functions._get_cached_tasks() self.assertIsInstance(tasks, dict) self.assertEqual(len(tasks), 3) self.assertIsInstance(list(tasks.values())[0], OpenMLTask) def test__get_cached_task(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir task = openml.tasks.functions._get_cached_task(1) self.assertIsInstance(task, OpenMLTask) def test__get_cached_task_not_cached(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir self.assertRaisesRegexp(OpenMLCacheException, 'Task file for tid 2 not cached', openml.tasks.functions._get_cached_task, 2) @@ -109,7 +109,7 @@ def test_list_tasks_per_type_paginate(self): self._check_task(tasks[tid]) def test__get_task(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir task = openml.tasks.get_task(1882) # Test the following task as it used to throw an Unicode Error. # https://github.com/openml/openml-python/issues/378 @@ -119,12 +119,15 @@ def test__get_task(self): def test_get_task(self): task = openml.tasks.get_task(1) self.assertIsInstance(task, OpenMLTask) - self.assertTrue(os.path.exists( - os.path.join(os.getcwd(), "tasks", "1", "task.xml"))) - self.assertTrue(os.path.exists( - os.path.join(os.getcwd(), "tasks", "1", "datasplits.arff"))) - self.assertTrue(os.path.exists( - os.path.join(os.getcwd(), "datasets", "1", "dataset.arff"))) + self.assertTrue(os.path.exists(os.path.join( + self.workdir, 'org', 'openml', 'test', "tasks", "1", "task.xml", + ))) + self.assertTrue(os.path.exists(os.path.join( + self.workdir, 'org', 'openml', 'test', "tasks", "1", "datasplits.arff" + ))) + self.assertTrue(os.path.exists(os.path.join( + self.workdir, 'org', 'openml', 'test', "datasets", "1", "dataset.arff" + ))) @mock.patch('openml.tasks.functions.get_dataset') def test_removal_upon_download_failure(self, get_dataset): @@ -145,7 +148,7 @@ def assert_and_raise(*args, **kwargs): )) def test_get_task_with_cache(self): - openml.config.set_cache_directory(self.static_cache_dir) + openml.config.cache_directory = self.static_cache_dir task = openml.tasks.get_task(1) self.assertIsInstance(task, OpenMLTask) @@ -153,13 +156,15 @@ def test_download_split(self): task = openml.tasks.get_task(1) split = task.download_split() self.assertEqual(type(split), OpenMLSplit) - self.assertTrue(os.path.exists( - os.path.join(os.getcwd(), "tasks", "1", "datasplits.arff"))) + self.assertTrue(os.path.exists(os.path.join( + self.workdir, 'org', 'openml', 'test', "tasks", "1", "datasplits.arff" + ))) def test_deletion_of_cache_dir(self): # Simple removal - tid_cache_dir = openml.tasks.functions.\ - _create_task_cache_directory(1) + tid_cache_dir = openml.utils._create_cache_directory_for_id( + 'tasks', 1, + ) self.assertTrue(os.path.exists(tid_cache_dir)) - openml.tasks.functions._remove_task_cache_dir(tid_cache_dir) + openml.utils._remove_cache_dir_for_id('tasks', tid_cache_dir) self.assertFalse(os.path.exists(tid_cache_dir)) From 9b9a8571a32877b0aab7fa69710f0834213e4fc2 Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Fri, 13 Apr 2018 10:37:28 +0200 Subject: [PATCH 83/86] Adding support for home directory symbol (#441) * Adding support for home directory symbol * Fixing backward compatibility --- openml/config.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/openml/config.py b/openml/config.py index 949fe869f..cb79da653 100644 --- a/openml/config.py +++ b/openml/config.py @@ -55,7 +55,7 @@ def _setup(): config = _parse_config() apikey = config.get('FAKE_SECTION', 'apikey') server = config.get('FAKE_SECTION', 'server') - cache_directory = config.get('FAKE_SECTION', 'cachedir') + cache_directory = os.path.expanduser(config.get('FAKE_SECTION', 'cachedir')) avoid_duplicate_runs = config.getboolean('FAKE_SECTION', 'avoid_duplicate_runs') @@ -105,8 +105,27 @@ def get_cache_directory(): return _cachedir +def set_cache_directory(cachedir): + """Set module-wide cache directory. + + Sets the cache directory into which to download datasets, tasks etc. + + Parameters + ---------- + cachedir : string + Path to use as cache directory. + + See also + -------- + get_cache_directory + """ + + global cache_directory + cache_directory = cachedir + + __all__ = [ - 'get_cache_directory', + 'get_cache_directory', 'set_cache_directory' ] _setup() From 67482f812e6d6382dd457cbac443dedcb966b2d1 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 13 Apr 2018 13:40:20 +0200 Subject: [PATCH 84/86] Add usage snippet to docs page (#434) * ADD introductory snippet to front page of the docs * FIX issues with rst formatting --- doc/index.rst | 38 +++++++++++++++++++++++++---- openml/__init__.py | 1 + openml/evaluations/evaluation.py | 41 ++++++++++++++++++-------------- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index ef2f0cd50..3990fc09a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -3,11 +3,39 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to OpenML's documentation! -================================== - -The OpenML module is still under development. You can watch its progress -:ref:`here `. +====== +OpenML +====== + +Welcome to the documentation of the OpenML Python API, a connector to the +collaborative machine learning platform `OpenML.org `_. +The OpenML Python package allows to use datasets and tasks from OpenML together +with scikit-learn and share the results online. + +------- +Example +------- + +.. code:: python + + # Define a scikit-learn pipeline + clf = sklearn.pipeline.Pipeline( + steps=[ + ('imputer', sklearn.preprocessing.Imputer()), + ('estimator', sklearn.tree.DecisionTreeClassifier()) + ] + ) + # Download the OpenML task for the german credit card dataset with 10-fold + # cross-validation. + task = openml.tasks.get_task(31) + # Set the OpenML API Key which is required to upload the runs. + # You can get your own API by signing up to OpenML.org. + openml.config.apikey = 'ABC' + # Run the scikit-learn model on the task (requires an API key). + run = openml.runs.run_model_on_task(task, clf) + # Publish the experiment on OpenML (optional, requires an API key). + run.publish() + print('URL for run: %s/run/%d' % (openml.config.server, run.run_id)) ------------ diff --git a/openml/__init__.py b/openml/__init__.py index b8c1fa6a8..d34f1bab6 100644 --- a/openml/__init__.py +++ b/openml/__init__.py @@ -28,6 +28,7 @@ from .runs import OpenMLRun from .tasks import OpenMLTask, OpenMLSplit from .flows import OpenMLFlow +from .evaluations import OpenMLEvaluation from .__version__ import __version__ diff --git a/openml/evaluations/evaluation.py b/openml/evaluations/evaluation.py index ad7466673..70acf0029 100644 --- a/openml/evaluations/evaluation.py +++ b/openml/evaluations/evaluation.py @@ -5,24 +5,29 @@ class OpenMLEvaluation(object): according to the evaluation/list function Parameters - ---------- - run_id : int - task_id : int - setup_id : int - flow_id : int - flow_name : str - data_id : int - data_name : str - the name of the dataset - function : str - the evaluation function of this item (e.g., accuracy) - upload_time : str - the time of evaluation - value : float - the value of this evaluation - array_data : str - list of information per class (e.g., in case of precision, auroc, - recall) + ---------- + run_id : int + + task_id : int + + setup_id : int + + flow_id : int + + flow_name : str + + data_id : int + + data_name : str + the name of the dataset + function : str + the evaluation function of this item (e.g., accuracy) + upload_time : str + the time of evaluation + value : float + the value of this evaluation + array_data : str + list of information per class (e.g., in case of precision, auroc, recall) ''' def __init__(self, run_id, task_id, setup_id, flow_id, flow_name, data_id, data_name, function, upload_time, value, From fdd6c2579704becbcecfc83d882f17f24090fdeb Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Sat, 21 Apr 2018 15:31:44 +0200 Subject: [PATCH 85/86] Fix unit tests (#448) * FIX fixes and improves unit tests * ADD additional asserts to hunt down bug * ADD debug output * remove print statement --- .gitignore | 1 + openml/runs/functions.py | 10 ++++++---- tests/test_runs/test_run_functions.py | 3 +++ tests/test_utils/__init__.py | 0 tests/test_utils/test_utils.py | 18 ++++++++++++++++++ 5 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 tests/test_utils/__init__.py create mode 100644 tests/test_utils/test_utils.py diff --git a/.gitignore b/.gitignore index 92a841500..4555e5cb6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*~ doc/generated examples/.ipynb_checkpoints # Byte-compiled / optimized / DLL files diff --git a/openml/runs/functions.py b/openml/runs/functions.py index e12c4ccd7..9e9697480 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -231,26 +231,28 @@ def _run_exists(task_id, setup_id): Parameters ---------- task_id: int + setup_id: int Returns ------- - List of run ids iff these already exists on the server, False otherwise + Set run ids for runs where flow setup_id was run on task_id. Empty + set if it wasn't run yet. """ if setup_id <= 0: # openml setups are in range 1-inf - return False + return set() try: result = list_runs(task=[task_id], setup=[setup_id]) if len(result) > 0: return set(result.keys()) else: - return False + return set() except OpenMLServerException as exception: # error code 512 implies no results. This means the run does not exist yet assert(exception.code == 512) - return False + return set() def _get_seeded_model(model, seed=None): diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index f824e1ed1..341900190 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -527,10 +527,13 @@ def test_get_run_trace(self): flow = openml.flows.sklearn_to_flow(clf) flow_exists = openml.flows.flow_exists(flow.name, flow.external_version) self.assertIsInstance(flow_exists, int) + self.assertGreater(flow_exists, 0) downloaded_flow = openml.flows.get_flow(flow_exists) setup_exists = openml.setups.setup_exists(downloaded_flow) self.assertIsInstance(setup_exists, int) + self.assertGreater(setup_exists, 0) run_ids = _run_exists(task.task_id, setup_exists) + self.assertGreater(len(run_ids), 0) run_id = random.choice(list(run_ids)) # now the actual unit test ... diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py new file mode 100644 index 000000000..9c5274810 --- /dev/null +++ b/tests/test_utils/test_utils.py @@ -0,0 +1,18 @@ +from openml.testing import TestBase +import openml + + +class OpenMLTaskTest(TestBase): + _multiprocess_can_split_ = True + + def test_list_all(self): + list_datasets = openml.datasets.functions._list_datasets + datasets = openml.utils.list_all(list_datasets) + + self.assertGreaterEqual(len(datasets), 100) + for did in datasets: + self._check_dataset(datasets[did]) + + # TODO implement these tests + # datasets = openml.utils.list_all(list_datasets, limit=50) + # self.assertEqual(len(datasets), 50) \ No newline at end of file From 96b0b8f6aa409ddb0023163b604b0e2340962bb8 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 23 Apr 2018 18:03:04 +0200 Subject: [PATCH 86/86] Fix circle-ci builds (#450) * FIX circle ci yaml file (remove requirements.txt) * circle ci: build docs on all branches --- circle.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/circle.yml b/circle.yml index cc61c92a1..ce5279bf1 100644 --- a/circle.yml +++ b/circle.yml @@ -25,14 +25,11 @@ dependencies: - pip install --upgrade pip - pip install --upgrade numpy - pip install --upgrade scipy - - pip install git+https://github.com/mfeurer/liac-arff.git # install documentation building dependencies - pip install --upgrade matplotlib setuptools nose coverage sphinx pillow sphinx-gallery sphinx_bootstrap_theme cython numpydoc scikit-learn nbformat nbconvert # Installing required packages for `make -C doc check command` to work. - sudo -E apt-get -yq update - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra - # finally install the requirements of the package to allow autodoc - - pip install -r requirements.txt # The --user is needed to let sphinx see the source and the binaries # The pipefail is requested to propagate exit code @@ -58,8 +55,3 @@ general: artifacts: - "doc/_build/html" - "~/log.txt" - # Restric the build to the branch master only - branches: - only: - - master - - develop