OpenML Benchmark generator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import openml as oml\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "import math\n",
    "from scipy.stats import norm\n",
    "from matplotlib import pyplot\n",
    "import sklearn.tree\n",
    "import sklearn.preprocessing\n",
    "from sklearn.externals.joblib import Parallel, delayed\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import StratifiedShuffleSplit, cross_val_score\n",
    "from openmlstudy14.preprocessing import ConditionalImputer\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Get datasets from OpenML\n",
    "- Only classification datasets\n",
    "- Only active (verified) datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1038 active classification datasets\n"
     ]
    }
   ],
   "source": [
    "# Get all OpenML datasets\n",
    "status_type = 'active'\n",
    "openml_list = oml.datasets.list_datasets(status=status_type) # Returns a dict\n",
    "datalist = pd.DataFrame.from_dict(openml_list, orient='index') # Transform to pandas\n",
    "#datalist = datalist[datalist.status == 'active'] # Only use active (verified) datasets\n",
    "datalist = datalist[datalist.NumberOfClasses>=2] # Only classification\n",
    "print(\"{} {} classification datasets\".format(len(datalist), status_type))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Bookkeeping\n",
    "data_names = {k: v for (k, v) in datalist[['did','name']].values} # dataset names\n",
    "data_status = {k: 'OK' for k in datalist.index} # dataset status (OK or reason for removal)\n",
    "datalist_full = datalist.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Apply simple preconditions\n",
    "- Number of observations larger than 500 (meaningful evaluations)\n",
    "- Number of observations smaller than 100000 (keep runtime manageable)\n",
    "- Number of features does not exceed 5000 (keep runtime manageable)\n",
    "- The ratio of the minority class and the majority class is > 0.05 (severely imbalanced datasets complicate analysis)\n",
    "- Number of values for categorical features must not exceed 100 (severely slows down some algorithms)\n",
    "- Sparsely formatted data (requires special data readers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Sparse format', 32],\n",
       " ['Extreme imbalance', 180],\n",
       " ['Too small', 343],\n",
       " ['OK', 313],\n",
       " ['High-dimensional', 60],\n",
       " ['Too large', 110]]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Apply preconditions\n",
    "data_status.update({k: 'Too small' for k in datalist.index[datalist.NumberOfInstances<500]})\n",
    "data_status.update({k: 'Too large' for k in datalist.index[datalist.NumberOfInstances>100000]})\n",
    "data_status.update({k: 'High-dimensional' for k in datalist.index[datalist.NumberOfFeatures>5000]})\n",
    "data_status.update({k: 'Extreme imbalance' for k in datalist.index[datalist.MinorityClassSize / datalist.MajorityClassSize < 0.05]})\n",
    "#data_status.update({k: 'Too many categories' for k in datalist.index[datalist.MaxNominalAttDistinctValues > 100]})\n",
    "data_status.update({k: 'Sparse format' for k in datalist.index[datalist.format == 'Sparse_ARFF']})\n",
    "\n",
    "# Filter dataset list\n",
    "datalist = datalist[pd.Series({k:(v=='OK') for k,v in data_status.items()})] \n",
    "\n",
    "# Status update\n",
    "[[x,list(data_status.values()).count(x)] for x in set(data_status.values())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Filter out special datasets\n",
    "- Artificial datasets (may bias the results)\n",
    "- Time series dataset (cannot use random sampling for evaluation)\n",
    "- Text data (contains string features which need additional preprocessing)\n",
    "- Multilabel data (multiple targets need to be predicted)\n",
    "- Derived versions of datasets (with additional preprocessing)\n",
    "- Datasets where the intended classification target is unclear\n",
    "- Binarized regression problems\n",
    "- Unknown origin (no description how data was collected and what the problem is)\n",
    "- Grouped data (instances form groups (blocks) and can't be randomly sampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vanrijn/projects/pythonvirtual/pimp/lib/python3.5/site-packages/ipykernel_launcher.py:26: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['Text data', 2],\n",
       " ['Time series data', 7],\n",
       " ['Sparse format', 32],\n",
       " ['Label leakage', 2],\n",
       " ['Extreme imbalance', 113],\n",
       " ['Too small', 342],\n",
       " ['Derived (non-original) data', 40],\n",
       " ['Binarized regression problem', 88],\n",
       " ['OK', 127],\n",
       " ['Artificial data', 203],\n",
       " ['High-dimensional', 60],\n",
       " ['Unspecified target feature', 5],\n",
       " ['Unknown origin', 8],\n",
       " ['Too large', 23],\n",
       " ['Grouped data', 2],\n",
       " ['Multi-label data', 7]]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get lists of special datasets\n",
    "artificial_set = set(oml.datasets.list_datasets(tag=\"artificial\", status=status_type).keys())\n",
    "timeseries_set = set(oml.datasets.list_datasets(tag=\"time_series\", status=status_type).keys())\n",
    "text_set = set(oml.datasets.list_datasets(tag=\"text_data\", status=status_type).keys())\n",
    "multilabel_set = set(oml.datasets.list_datasets(tag=\"multi_label\", status=status_type).keys())\n",
    "derived_set = set(oml.datasets.list_datasets(tag=\"derived\", status=status_type).keys())\n",
    "unspecified_set = set(oml.datasets.list_datasets(tag=\"unspecified_target_feature\", status=status_type).keys())\n",
    "binarized_set = set(oml.datasets.list_datasets(tag=\"binarized_regression_problem\", status=status_type).keys())\n",
    "unknown_set = set(oml.datasets.list_datasets(tag=\"origin_unknown\", status=status_type).keys())\n",
    "grouped_set = set(oml.datasets.list_datasets(tag=\"grouped_data\", status=status_type).keys())\n",
    "label_leakage = set(oml.datasets.list_datasets(tag=\"label_leakage\", status=status_type).keys())\n",
    "single_feature = set(oml.datasets.list_datasets(tag=\"single_feature_predictable\", status=status_type).keys())\n",
    "\n",
    "data_status.update({k: 'Artificial data' for k in artificial_set})\n",
    "data_status.update({k: 'Time series data' for k in timeseries_set})\n",
    "data_status.update({k: 'Text data' for k in text_set})\n",
    "data_status.update({k: 'Multi-label data' for k in multilabel_set})\n",
    "data_status.update({k: 'Derived (non-original) data' for k in derived_set})\n",
    "data_status.update({k: 'Unspecified target feature' for k in unspecified_set})\n",
    "data_status.update({k: 'Binarized regression problem' for k in binarized_set})\n",
    "data_status.update({k: 'Unknown origin' for k in unknown_set})\n",
    "data_status.update({k: 'Grouped data' for k in grouped_set})\n",
    "data_status.update({k: 'Label leakage' for k in label_leakage})\n",
    "data_status.update({k: 'Single feature' for k in single_feature})\n",
    "#data_status.update({k: 'OpenML100' for k in openml100_set})\n",
    "\n",
    "# Filter dataset list\n",
    "datalist = datalist[pd.Series({k:(v=='OK') for k,v in data_status.items()})] \n",
    "\n",
    "# Status update\n",
    "[[x,list(data_status.values()).count(x)] for x in set(data_status.values())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Remove alternative versions of datasets\n",
    "- Remove binarized versions of multi-class datasets\n",
    "- Check other possible duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vanrijn/projects/pythonvirtual/pimp/lib/python3.5/site-packages/ipykernel_launcher.py:43: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['Time series data', 7],\n",
       " ['Binarized version of multiclass dataset', 77],\n",
       " ['Too small', 301],\n",
       " ['Grouped data', 2],\n",
       " ['Text data', 2],\n",
       " ['Label leakage', 2],\n",
       " ['Derived (non-original) data', 38],\n",
       " ['OK', 89],\n",
       " ['Duplicate of 772', 1],\n",
       " ['Duplicate of 1590', 1],\n",
       " ['Unspecified target feature', 5],\n",
       " ['Multi-label data', 6],\n",
       " ['Duplicate of 40984', 2],\n",
       " ['Duplicate of 40994', 1],\n",
       " ['Extreme imbalance', 112],\n",
       " ['High-dimensional', 60],\n",
       " ['Too large', 23],\n",
       " ['Sparse format', 31],\n",
       " ['Duplicate of 40979', 2],\n",
       " ['Duplicate of 40982', 1],\n",
       " ['Duplicate of 40945', 1],\n",
       " ['Binarized regression problem', 86],\n",
       " ['Artificial data', 202],\n",
       " ['Duplicate of 40983', 1],\n",
       " ['Unknown origin', 8]]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sorting makes things easier\n",
    "# We need the full list because there may be binarized versions of already removed datasets\n",
    "datalist_full = datalist_full.sort_values(by=['name','NumberOfClasses'], ascending=[True, False])\n",
    "\n",
    "checked_datasets = {\n",
    "    40979: [1022, 20], # Correct version of mfeat-pixel\n",
    "    40984: [958, 36], # Correct version of segment\n",
    "    40994: [40990, 40989, 1467], # Correct version of climate-model-simulation-crashes\n",
    "    1590: [179], # Correct version of adult\n",
    "    40983: [1570], # Correct version of wilt\n",
    "    40945: [40704], # Correct version of Titanic\n",
    "    772: [948], # Correct version of classification version of the quake dataset\n",
    "    40966: [40965, 40964], # Correct version of MiceProtein (needed when including in_preparation datasets)\n",
    "    40982: [40973, 1504], # Correct version of steel-plates-fault (needed when including in_preparation datasets)\n",
    "    23380: [1024, 473], # Correct version of cjs (needed when including in_preparation datasets)\n",
    "    40597 : [40733], # Correct version of yeast (needed when including in_preparation datasets)\n",
    "    1046 : [40829], # Correct version of mozilla4 (needed when including in_preparation datasets)\n",
    "    # : [40958], # Correct version of Bankdata (needed when including in_preparation datasets)\n",
    "    \n",
    "}\n",
    "duplicates_of = {}\n",
    "# Mark the duplicates of datasets where we know which version is the correct one!\n",
    "for cd in checked_datasets:\n",
    "    for dup_id in checked_datasets[cd]:\n",
    "        duplicates_of[dup_id] = cd\n",
    "\n",
    "data_unique = {}\n",
    "for index, row in datalist_full.iterrows():\n",
    "    if row['did'] in duplicates_of:\n",
    "        data_status[row['did']] = 'Duplicate of %d' % duplicates_of[row['did']]\n",
    "    elif row['did'] in checked_datasets and data_status[row['did']] in ('OK', 'Possible duplicate'):\n",
    "        data_status[row['did']] = 'OK'\n",
    "    elif row['name'] not in data_unique:\n",
    "        data_unique[row['name']] = row\n",
    "    else:\n",
    "        previous = data_unique[row['name']]\n",
    "        if previous['NumberOfClasses'] > 2 and row['NumberOfClasses'] == 2:\n",
    "            data_status[row['did']] = 'Binarized version of multiclass dataset'\n",
    "        elif data_status[row['did']] in ('OK', 'Possible duplicate'):\n",
    "            data_status[row['did']] = 'Possible duplicate'\n",
    "\n",
    "# Filter dataset list\n",
    "datalist = datalist[pd.Series({k:(v=='OK') for k,v in data_status.items()})] \n",
    "               \n",
    "# Status update\n",
    "[[x,list(data_status.values()).count(x)] for x in set(data_status.values())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# These need to be checked\n",
    "[k for k,v in data_status.items() if v=='Possible duplicate']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Remove trivial datasets\n",
    "- See if a model (e.g. random forest) based on 1 feature can get perfect CV performance (JvR, removed this code. But Irish and cjs came out of this check. I will tag them. )\n",
    "- See if a CV Decision Tree (flow id 7777) or a CV logistic regression (flow id 7778) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Time series data', 7],\n",
       " ['Binarized version of multiclass dataset', 77],\n",
       " ['Too easy (perfect score decision tree)', 2],\n",
       " ['Too small', 301],\n",
       " ['Dataset not checked for triviality by DT!', 3],\n",
       " ['Grouped data', 2],\n",
       " ['Text data', 2],\n",
       " ['Label leakage', 2],\n",
       " ['Derived (non-original) data', 38],\n",
       " ['OK', 84],\n",
       " ['Duplicate of 772', 1],\n",
       " ['Duplicate of 1590', 1],\n",
       " ['Unspecified target feature', 5],\n",
       " ['Multi-label data', 6],\n",
       " ['Duplicate of 40984', 2],\n",
       " ['Duplicate of 40994', 1],\n",
       " ['Extreme imbalance', 112],\n",
       " ['High-dimensional', 60],\n",
       " ['Too large', 23],\n",
       " ['Sparse format', 31],\n",
       " ['Duplicate of 40979', 2],\n",
       " ['Duplicate of 40982', 1],\n",
       " ['Duplicate of 40945', 1],\n",
       " ['Binarized regression problem', 86],\n",
       " ['Artificial data', 202],\n",
       " ['Duplicate of 40983', 1],\n",
       " ['Unknown origin', 8]]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perfect_scores_dt = {int(v.data_id) for k, v in oml.evaluations.list_evaluations('predictive_accuracy', flow=[7777]).items() if v.value == 1}\n",
    "perfect_scores_lr = {int(v.data_id) for k, v in oml.evaluations.list_evaluations('predictive_accuracy', flow=[7778]).items() if v.value == 1}\n",
    "\n",
    "for did in perfect_scores_dt:\n",
    "    if data_status[did] == 'OK':\n",
    "        data_status[did] = 'Too easy (perfect score decision tree)'\n",
    "        \n",
    "for did in perfect_scores_lr:\n",
    "    if data_status[did] == 'OK':\n",
    "        data_status[did] = 'Too easy (perfect score logistic regression)'\n",
    "     \n",
    "checked_datasets = {int(v.data_id) for v in oml.evaluations.list_evaluations('predictive_accuracy', flow=[7777]).values()}\n",
    "for did in data_status:\n",
    "    if data_status[did] == 'OK' and did not in checked_datasets:\n",
    "        data_status[did] = 'Dataset not checked for triviality by DT!'\n",
    "checked_datasets = {int(v.data_id) for v in oml.evaluations.list_evaluations('predictive_accuracy', flow=[7778]).values()}\n",
    "# for did in data_status:\n",
    "#     if data_status[did] == 'OK' and did not in checked_datasets:\n",
    "#         data_status[did] = 'Dataset not checked for triviality by LR!'\n",
    "\n",
    "[[x,list(data_status.values()).count(x)] for x in set(data_status.values())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing dataset 3 kr-vs-kp ( 1 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 6 letter ( 2 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 11 balance-scale ( 3 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 12 mfeat-factors ( 4 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 14 mfeat-fourier ( 5 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 15 breast-w ( 6 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 16 mfeat-karhunen ( 7 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 18 mfeat-morphological ( 8 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40979 mfeat-pixel ( 9 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40981 Australian ( 10 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 22 mfeat-zernike ( 11 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 23 cmc ( 12 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 28 optdigits ( 13 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 29 credit-approval ( 14 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 31 credit-g ( 15 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 32 pendigits ( 16 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 37 diabetes ( 17 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 38 sick ( 18 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 42 soybean ( 19 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 44 spambase ( 20 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 46 splice ( 21 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 50 tic-tac-toe ( 22 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 54 vehicle ( 23 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 60 waveform-5000 ( 24 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40971 collins ( 25 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40984 segment ( 26 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 151 electricity ( 27 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 182 satimage ( 28 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 188 eucalyptus ( 29 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40994 climate-model-simulation-crashes ( 30 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40996 Fashion-MNIST ( 31 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 4134 Bioresponse ( 32 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 4135 Amazon_employee_access ( 33 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "Amazon_employee_access too high one-hot-encoded dimensionality 15626\n",
      "processing dataset 300 isolet ( 34 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 307 vowel ( 35 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 41027 jungle_chess_2pcs_raw_endgame_complete ( 36 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 4534 PhishingWebsites ( 37 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 4538 GesturePhaseSegmentationProcessed ( 38 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 469 analcatdata_dmft ( 39 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 554 mnist_784 ( 40 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1049 pc4 ( 41 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1050 pc3 ( 42 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1053 jm1 ( 43 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1063 kc2 ( 44 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1067 kc1 ( 45 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1068 pc1 ( 46 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1112 KDDCup09_churn ( 47 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "KDDCup09_churn too high one-hot-encoded dimensionality 71696\n",
      "processing dataset 1114 KDDCup09_upselling ( 48 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "KDDCup09_upselling too high one-hot-encoded dimensionality 71696\n",
      "processing dataset 1116 musk ( 49 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "Dataset  musk is too easy.\n",
      "processing dataset 1120 MagicTelescope ( 50 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40966 MiceProtein ( 51 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1461 bank-marketing ( 52 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1462 banknote-authentication ( 53 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1464 blood-transfusion-service-center ( 54 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1468 cnae-9 ( 55 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1475 first-order-theorem-proving ( 56 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1478 har ( 57 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1480 ilpd ( 58 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1485 madelon ( 59 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1486 nomao ( 60 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1487 ozone-level-8hr ( 61 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1489 phoneme ( 62 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1491 one-hundred-plants-margin ( 63 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1492 one-hundred-plants-shape ( 64 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1493 one-hundred-plants-texture ( 65 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1494 qsar-biodeg ( 66 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1497 wall-robot-navigation ( 67 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1501 semeion ( 68 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1510 wdbc ( 69 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1515 micro-mass ( 70 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40499 texture ( 71 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 1590 adult ( 72 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40536 SpeedDating ( 73 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 6332 cylinder-bands ( 74 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 23381 dresses-sales ( 75 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40668 connect-4 ( 76 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40670 dna ( 77 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40701 churn ( 78 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40705 tokyo1 ( 79 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 23512 higgs ( 80 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 23517 numerai28.6 ( 81 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40982 steel-plates-fault ( 82 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40983 wilt ( 83 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n",
      "processing dataset 40923 Devnagari-Script ( 84 / 84 )\n",
      "One hot encoding dataset to check for the true amount of total features.\n",
      "building decision stump\n",
      "obtaining cv task .. \n"
     ]
    }
   ],
   "source": [
    "datasets = [k for k,v in data_status.items() if v=='OK']\n",
    "\n",
    "max_score_per_dataset = {}\n",
    "for idx, dataset_id in enumerate(datasets):\n",
    "    try:\n",
    "        \n",
    "        dataset = oml.datasets.get_dataset(dataset_id)\n",
    "        print('processing dataset', dataset_id, dataset.name, '(',idx+1, '/', len(datasets), ')')\n",
    "        if dataset.default_target_attribute is None:\n",
    "            data_status[dataset_id] = 'No target specified'\n",
    "            print('No target')\n",
    "            continue\n",
    "        X, y = dataset.get_data(target=dataset.default_target_attribute)\n",
    "\n",
    "        print('One hot encoding dataset to check for the true amount of total features.')\n",
    "        categorical_indices = dataset.get_features_by_type('nominal', [dataset.default_target_attribute])\n",
    "        clf = sklearn.pipeline.Pipeline(\n",
    "            steps=[\n",
    "                (\n",
    "                    'imputer', ConditionalImputer(\n",
    "                        strategy='median', \n",
    "                        strategy_nominal='mean', \n",
    "                        categorical_features=categorical_indices, \n",
    "                        fill_empty=-1,\n",
    "                    )\n",
    "                ), \n",
    "                (\n",
    "                    'encoder', \n",
    "                    sklearn.preprocessing.OneHotEncoder(categorical_features=categorical_indices)\n",
    "                )\n",
    "            ]\n",
    "        )\n",
    "        hotencoded = clf.fit_transform(X)\n",
    "        if hotencoded.shape[1] > 5000:\n",
    "            data_status[dataset_id] = 'High-dimensional (after one hot encoding.)'\n",
    "            print(dataset.name, 'too high one-hot-encoded dimensionality', hotencoded.shape[1])\n",
    "            continue\n",
    "\n",
    "        print('building decision stump')\n",
    "        clf = sklearn.pipeline.Pipeline(steps=[('imputer', sklearn.preprocessing.Imputer(strategy='median')), \n",
    "                                               ('classifier', sklearn.tree.DecisionTreeClassifier(max_depth=1))])\n",
    "        _ = clf.fit(X, y)\n",
    "        score = clf.score(X, y)\n",
    "        \n",
    "        print('obtaining cv task .. ')\n",
    "        # TODO\n",
    "\n",
    "        max_score_per_dataset[dataset_id] = {\n",
    "            'score': score,\n",
    "            'name': dataset.name\n",
    "        }\n",
    "    except ValueError as e:\n",
    "        data_status[dataset_id] = 'Python ValueError'\n",
    "        print(dataset_id, e)\n",
    "        continue\n",
    "    except Exception as e:\n",
    "        data_status[dataset_id] = 'Python Exception'\n",
    "        print(dataset_id, e)\n",
    "        continue\n",
    "    \n",
    "    \n",
    "    if max_score_per_dataset[dataset_id][\"score\"] == 1.00:\n",
    "        data_status[dataset_id] = 'Too easy (decisionstump on trainset)'\n",
    "        print(\"Dataset \", dataset.name, \"is too easy.\")\n",
    "    \n",
    "results = pd.DataFrame(max_score_per_dataset).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1493</th>\n",
       "      <td>one-hundred-plants-texture</td>\n",
       "      <td>0.0193871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1491</th>\n",
       "      <td>one-hundred-plants-margin</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1492</th>\n",
       "      <td>one-hundred-plants-shape</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40923</th>\n",
       "      <td>Devnagari-Script</td>\n",
       "      <td>0.0401522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>letter</td>\n",
       "      <td>0.0718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>isolet</td>\n",
       "      <td>0.0766962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40971</th>\n",
       "      <td>collins</td>\n",
       "      <td>0.109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>vowel</td>\n",
       "      <td>0.167677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40499</th>\n",
       "      <td>texture</td>\n",
       "      <td>0.180545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1515</th>\n",
       "      <td>micro-mass</td>\n",
       "      <td>0.182137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>mfeat-zernike</td>\n",
       "      <td>0.186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>mfeat-karhunen</td>\n",
       "      <td>0.189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>mfeat-fourier</td>\n",
       "      <td>0.1915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mfeat-factors</td>\n",
       "      <td>0.192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>optdigits</td>\n",
       "      <td>0.195018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40979</th>\n",
       "      <td>mfeat-pixel</td>\n",
       "      <td>0.1975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1501</th>\n",
       "      <td>semeion</td>\n",
       "      <td>0.19774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554</th>\n",
       "      <td>mnist_784</td>\n",
       "      <td>0.198743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40996</th>\n",
       "      <td>Fashion-MNIST</td>\n",
       "      <td>0.199229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>mfeat-morphological</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>pendigits</td>\n",
       "      <td>0.204785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1468</th>\n",
       "      <td>cnae-9</td>\n",
       "      <td>0.205556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>analcatdata_dmft</td>\n",
       "      <td>0.212045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>soybean</td>\n",
       "      <td>0.234261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40966</th>\n",
       "      <td>MiceProtein</td>\n",
       "      <td>0.273148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40984</th>\n",
       "      <td>segment</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1478</th>\n",
       "      <td>har</td>\n",
       "      <td>0.373823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>vehicle</td>\n",
       "      <td>0.410165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1475</th>\n",
       "      <td>first-order-theorem-proving</td>\n",
       "      <td>0.417457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>satimage</td>\n",
       "      <td>0.424417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>diabetes</td>\n",
       "      <td>0.735677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4134</th>\n",
       "      <td>Bioresponse</td>\n",
       "      <td>0.73687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1494</th>\n",
       "      <td>qsar-biodeg</td>\n",
       "      <td>0.745972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1489</th>\n",
       "      <td>phoneme</td>\n",
       "      <td>0.754626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>electricity</td>\n",
       "      <td>0.757393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1590</th>\n",
       "      <td>adult</td>\n",
       "      <td>0.760718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1464</th>\n",
       "      <td>blood-transfusion-service-center</td>\n",
       "      <td>0.762032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>spambase</td>\n",
       "      <td>0.79374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053</th>\n",
       "      <td>jm1</td>\n",
       "      <td>0.806523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40536</th>\n",
       "      <td>SpeedDating</td>\n",
       "      <td>0.835283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1067</th>\n",
       "      <td>kc1</td>\n",
       "      <td>0.845424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4534</th>\n",
       "      <td>PhishingWebsites</td>\n",
       "      <td>0.847309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>kc2</td>\n",
       "      <td>0.850575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1486</th>\n",
       "      <td>nomao</td>\n",
       "      <td>0.852807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1462</th>\n",
       "      <td>banknote-authentication</td>\n",
       "      <td>0.853499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>credit-approval</td>\n",
       "      <td>0.855072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40981</th>\n",
       "      <td>Australian</td>\n",
       "      <td>0.855072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40701</th>\n",
       "      <td>churn</td>\n",
       "      <td>0.8706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1049</th>\n",
       "      <td>pc4</td>\n",
       "      <td>0.877915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1461</th>\n",
       "      <td>bank-marketing</td>\n",
       "      <td>0.883015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050</th>\n",
       "      <td>pc3</td>\n",
       "      <td>0.897633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40705</th>\n",
       "      <td>tokyo1</td>\n",
       "      <td>0.89781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40994</th>\n",
       "      <td>climate-model-simulation-crashes</td>\n",
       "      <td>0.914815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1510</th>\n",
       "      <td>wdbc</td>\n",
       "      <td>0.922671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>breast-w</td>\n",
       "      <td>0.924177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1068</th>\n",
       "      <td>pc1</td>\n",
       "      <td>0.935978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1487</th>\n",
       "      <td>ozone-level-8hr</td>\n",
       "      <td>0.936859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40983</th>\n",
       "      <td>wilt</td>\n",
       "      <td>0.946063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>sick</td>\n",
       "      <td>0.965536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1116</th>\n",
       "      <td>musk</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>81 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   name      score\n",
       "1493         one-hundred-plants-texture  0.0193871\n",
       "1491          one-hundred-plants-margin       0.02\n",
       "1492           one-hundred-plants-shape       0.02\n",
       "40923                  Devnagari-Script  0.0401522\n",
       "6                                letter     0.0718\n",
       "300                              isolet  0.0766962\n",
       "40971                           collins      0.109\n",
       "307                               vowel   0.167677\n",
       "40499                           texture   0.180545\n",
       "1515                         micro-mass   0.182137\n",
       "22                        mfeat-zernike      0.186\n",
       "16                       mfeat-karhunen      0.189\n",
       "14                        mfeat-fourier     0.1915\n",
       "12                        mfeat-factors      0.192\n",
       "28                            optdigits   0.195018\n",
       "40979                       mfeat-pixel     0.1975\n",
       "1501                            semeion    0.19774\n",
       "554                           mnist_784   0.198743\n",
       "40996                     Fashion-MNIST   0.199229\n",
       "18                  mfeat-morphological        0.2\n",
       "32                            pendigits   0.204785\n",
       "1468                             cnae-9   0.205556\n",
       "469                    analcatdata_dmft   0.212045\n",
       "42                              soybean   0.234261\n",
       "40966                       MiceProtein   0.273148\n",
       "40984                           segment   0.285714\n",
       "1478                                har   0.373823\n",
       "54                              vehicle   0.410165\n",
       "1475        first-order-theorem-proving   0.417457\n",
       "182                            satimage   0.424417\n",
       "...                                 ...        ...\n",
       "37                             diabetes   0.735677\n",
       "4134                        Bioresponse    0.73687\n",
       "1494                        qsar-biodeg   0.745972\n",
       "1489                            phoneme   0.754626\n",
       "151                         electricity   0.757393\n",
       "1590                              adult   0.760718\n",
       "1464   blood-transfusion-service-center   0.762032\n",
       "44                             spambase    0.79374\n",
       "1053                                jm1   0.806523\n",
       "40536                       SpeedDating   0.835283\n",
       "1067                                kc1   0.845424\n",
       "4534                   PhishingWebsites   0.847309\n",
       "1063                                kc2   0.850575\n",
       "1486                              nomao   0.852807\n",
       "1462            banknote-authentication   0.853499\n",
       "29                      credit-approval   0.855072\n",
       "40981                        Australian   0.855072\n",
       "40701                             churn     0.8706\n",
       "1049                                pc4   0.877915\n",
       "1461                     bank-marketing   0.883015\n",
       "1050                                pc3   0.897633\n",
       "40705                            tokyo1    0.89781\n",
       "40994  climate-model-simulation-crashes   0.914815\n",
       "1510                               wdbc   0.922671\n",
       "15                             breast-w   0.924177\n",
       "1068                                pc1   0.935978\n",
       "1487                    ozone-level-8hr   0.936859\n",
       "40983                              wilt   0.946063\n",
       "38                                 sick   0.965536\n",
       "1116                               musk          1\n",
       "\n",
       "[81 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.sort_values(by='score')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5 Remove datasets for other reasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_status[40705] = 'Missing description.'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results\n",
    "Final list of selected datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79 datasets selected\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{3: 'kr-vs-kp',\n",
       " 6: 'letter',\n",
       " 11: 'balance-scale',\n",
       " 12: 'mfeat-factors',\n",
       " 14: 'mfeat-fourier',\n",
       " 15: 'breast-w',\n",
       " 16: 'mfeat-karhunen',\n",
       " 18: 'mfeat-morphological',\n",
       " 22: 'mfeat-zernike',\n",
       " 23: 'cmc',\n",
       " 28: 'optdigits',\n",
       " 29: 'credit-approval',\n",
       " 31: 'credit-g',\n",
       " 32: 'pendigits',\n",
       " 37: 'diabetes',\n",
       " 38: 'sick',\n",
       " 42: 'soybean',\n",
       " 44: 'spambase',\n",
       " 46: 'splice',\n",
       " 50: 'tic-tac-toe',\n",
       " 54: 'vehicle',\n",
       " 60: 'waveform-5000',\n",
       " 151: 'electricity',\n",
       " 182: 'satimage',\n",
       " 188: 'eucalyptus',\n",
       " 300: 'isolet',\n",
       " 307: 'vowel',\n",
       " 469: 'analcatdata_dmft',\n",
       " 554: 'mnist_784',\n",
       " 1049: 'pc4',\n",
       " 1050: 'pc3',\n",
       " 1053: 'jm1',\n",
       " 1063: 'kc2',\n",
       " 1067: 'kc1',\n",
       " 1068: 'pc1',\n",
       " 1120: 'MagicTelescope',\n",
       " 1461: 'bank-marketing',\n",
       " 1462: 'banknote-authentication',\n",
       " 1464: 'blood-transfusion-service-center',\n",
       " 1468: 'cnae-9',\n",
       " 1475: 'first-order-theorem-proving',\n",
       " 1478: 'har',\n",
       " 1480: 'ilpd',\n",
       " 1485: 'madelon',\n",
       " 1486: 'nomao',\n",
       " 1487: 'ozone-level-8hr',\n",
       " 1489: 'phoneme',\n",
       " 1491: 'one-hundred-plants-margin',\n",
       " 1492: 'one-hundred-plants-shape',\n",
       " 1493: 'one-hundred-plants-texture',\n",
       " 1494: 'qsar-biodeg',\n",
       " 1497: 'wall-robot-navigation',\n",
       " 1501: 'semeion',\n",
       " 1510: 'wdbc',\n",
       " 1515: 'micro-mass',\n",
       " 1590: 'adult',\n",
       " 4134: 'Bioresponse',\n",
       " 4534: 'PhishingWebsites',\n",
       " 4538: 'GesturePhaseSegmentationProcessed',\n",
       " 6332: 'cylinder-bands',\n",
       " 23381: 'dresses-sales',\n",
       " 23512: 'higgs',\n",
       " 23517: 'numerai28.6',\n",
       " 40499: 'texture',\n",
       " 40536: 'SpeedDating',\n",
       " 40668: 'connect-4',\n",
       " 40670: 'dna',\n",
       " 40701: 'churn',\n",
       " 40923: 'Devnagari-Script',\n",
       " 40966: 'MiceProtein',\n",
       " 40971: 'collins',\n",
       " 40979: 'mfeat-pixel',\n",
       " 40981: 'Australian',\n",
       " 40982: 'steel-plates-fault',\n",
       " 40983: 'wilt',\n",
       " 40984: 'segment',\n",
       " 40994: 'climate-model-simulation-crashes',\n",
       " 40996: 'Fashion-MNIST',\n",
       " 41027: 'jungle_chess_2pcs_raw_endgame_complete'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_datasets = [k for k,v in data_status.items() if v=='OK']\n",
    "print('{} datasets selected'.format(len(final_datasets)))\n",
    "{k:v for k,v in data_names.items() if k in final_datasets}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Passed all tests, but not in OpenML100:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{23517: 'numerai28.6',\n",
       " 40670: 'dna',\n",
       " 40701: 'churn',\n",
       " 40923: 'Devnagari-Script',\n",
       " 40966: 'MiceProtein',\n",
       " 40971: 'collins',\n",
       " 40979: 'mfeat-pixel',\n",
       " 40982: 'steel-plates-fault',\n",
       " 40983: 'wilt',\n",
       " 40984: 'segment',\n",
       " 40994: 'climate-model-simulation-crashes',\n",
       " 40996: 'Fashion-MNIST',\n",
       " 41027: 'jungle_chess_2pcs_raw_endgame_complete'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "openml100_set = set(oml.datasets.list_datasets(tag=\"OpenML100\").keys()) # OpenML100\n",
    "\n",
    "new_datasets = [k for k,v in data_status.items() if v=='OK' and k not in openml100_set]\n",
    "{k:v for k,v in data_names.items() if k in new_datasets}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Datasets tagged with OpenML100 that did not pass all tests:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{20: 'mfeat-pixel',\n",
       " 24: 'mushroom',\n",
       " 36: 'segment',\n",
       " 312: 'scene',\n",
       " 333: 'monks-problems-1',\n",
       " 334: 'monks-problems-2',\n",
       " 335: 'monks-problems-3',\n",
       " 375: 'JapaneseVowels',\n",
       " 377: 'synthetic_control',\n",
       " 451: 'irish',\n",
       " 458: 'analcatdata_authorship',\n",
       " 470: 'profb',\n",
       " 1038: 'gina_agnostic',\n",
       " 1046: 'mozilla4',\n",
       " 1112: 'KDDCup09_churn',\n",
       " 1114: 'KDDCup09_upselling',\n",
       " 1459: 'artificial-characters',\n",
       " 1466: 'cardiotocography',\n",
       " 1467: 'climate-model-simulation-crashes',\n",
       " 1471: 'eeg-eye-state',\n",
       " 1476: 'gas-drift',\n",
       " 1479: 'hill-valley',\n",
       " 1504: 'steel-plates-fault',\n",
       " 1570: 'wilt',\n",
       " 4135: 'Amazon_employee_access',\n",
       " 23380: 'cjs',\n",
       " 40496: 'LED-display-domain-7digit'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_datasets = [k for k,v in data_status.items() if k in openml100_set and v!='OK']\n",
    "{k:v for k,v in data_names.items() if k in new_datasets}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reasons to exclude datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Artificial data': '[70, 71, 72, 73, 74, 75, 76, 77, 78, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 146, 147, 148, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 271, 272, 333, 334, 335, 1177, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1459, 1460, 1479, 1496, 1507, 1547, 1548, 1549, 1551, 1552, 1553, 1554, 1555, 40496, 40514, 40515, 40516, 40518, 40519, 40520, 40645, 40646, 40647, 40648, 40649, 40650, 40677, 40678, 40680, 40690, 40693, 40706]',\n",
       " 'Binarized regression problem': '[715, 717, 718, 722, 723, 725, 727, 728, 734, 735, 737, 740, 742, 743, 749, 750, 751, 752, 757, 761, 766, 770, 772, 774, 779, 792, 795, 797, 799, 802, 803, 805, 806, 807, 813, 816, 819, 821, 823, 824, 825, 826, 827, 833, 837, 838, 839, 841, 843, 845, 846, 847, 849, 853, 855, 866, 869, 870, 871, 879, 881, 884, 886, 888, 896, 901, 903, 904, 910, 912, 913, 914, 917, 920, 923, 926, 930, 931, 934, 936, 937, 940, 943, 949, 981, 993]',\n",
       " 'Binarized version of multiclass dataset': '[293, 720, 741, 818, 867, 887, 897, 953, 954, 955, 957, 959, 960, 961, 962, 963, 964, 965, 966, 967, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 982, 983, 984, 985, 986, 987, 988, 989, 990, 992, 994, 995, 996, 997, 998, 999, 1000, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1023, 1025, 1026, 1178, 1524, 40597, 40683, 40702]',\n",
       " 'Dataset not checked for triviality by DT!': '[1038, 40927, 40978]',\n",
       " 'Derived (non-original) data': '[378, 381, 382, 991, 1037, 1040, 1041, 1042, 1119, 1217, 1218, 1219, 1220, 1223, 1226, 1525, 1526, 1557, 1558, 1560, 1566, 1568, 40664, 40666, 40926, 40992, 40993, 40997, 40998, 40999, 41000, 41001, 41002, 41003, 41004, 41005, 41006, 41007]',\n",
       " 'Duplicate of 1590': '[179]',\n",
       " 'Duplicate of 40945': '[40704]',\n",
       " 'Duplicate of 40979': '[20, 1022]',\n",
       " 'Duplicate of 40982': '[1504]',\n",
       " 'Duplicate of 40983': '[1570]',\n",
       " 'Duplicate of 40984': '[36, 958]',\n",
       " 'Duplicate of 40994': '[1467]',\n",
       " 'Duplicate of 772': '[948]',\n",
       " 'Extreme imbalance': '[2, 5, 7, 8, 9, 10, 26, 30, 34, 39, 41, 45, 47, 49, 51, 57, 149, 150, 155, 171, 180, 181, 183, 184, 185, 186, 275, 279, 310, 311, 313, 316, 329, 343, 372, 374, 379, 443, 449, 453, 454, 477, 488, 947, 950, 951, 1039, 1056, 1069, 1088, 1102, 1109, 1110, 1111, 1113, 1142, 1146, 1216, 1414, 1472, 1481, 1483, 1509, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1567, 1569, 1596, 1597, 4154, 4340, 4537, 4546, 4552, 40474, 40475, 40476, 40477, 40478, 40497, 40498, 40588, 40590, 40593, 40596, 40672, 40685, 40691, 40707, 40708, 40713, 40753, 41021, 41022]',\n",
       " 'Grouped data': '[375, 1044]',\n",
       " 'High-dimensional': '[1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1104, 1106, 1107, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1143, 1144, 1145, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1233, 1457, 1458]',\n",
       " 'High-dimensional (after one hot encoding.)': '[1112, 1114, 4135]',\n",
       " 'Label leakage': '[451, 23380]',\n",
       " 'Missing description.': '[40705]',\n",
       " 'Multi-label data': '[312, 40589, 40591, 40592, 40594, 40595]',\n",
       " 'OK': '[3, 6, 11, 12, 14, 15, 16, 18, 22, 23, 28, 29, 31, 32, 37, 38, 42, 44, 46, 50, 54, 60, 151, 182, 188, 300, 307, 469, 554, 1049, 1050, 1053, 1063, 1067, 1068, 1120, 1461, 1462, 1464, 1468, 1475, 1478, 1480, 1485, 1486, 1487, 1489, 1491, 1492, 1493, 1494, 1497, 1501, 1510, 1515, 1590, 4134, 4534, 4538, 6332, 23381, 23512, 23517, 40499, 40536, 40668, 40670, 40701, 40923, 40966, 40971, 40979, 40981, 40982, 40983, 40984, 40994, 40996, 41027]',\n",
       " 'Sparse format': '[273, 350, 351, 354, 357, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 1241, 1242, 4136, 4137, 4139, 4140, 41026]',\n",
       " 'Text data': '[373, 40945]',\n",
       " 'Time series data': '[377, 1046, 1471, 1476, 1477, 40922, 40985]',\n",
       " 'Too easy (decisionstump on trainset)': '[1116]',\n",
       " 'Too easy (perfect score decision tree)': '[24, 1466]',\n",
       " 'Too large': '[152, 153, 154, 156, 157, 158, 159, 160, 161, 162, 274, 1169, 1235, 1236, 1237, 1238, 1240, 1502, 1503, 4535, 4541, 23383, 40517]',\n",
       " 'Too small': '[4, 13, 25, 27, 35, 40, 43, 48, 52, 53, 55, 56, 59, 61, 62, 163, 164, 165, 166, 167, 168, 169, 170, 172, 175, 176, 177, 178, 187, 190, 276, 277, 278, 285, 327, 328, 336, 337, 338, 339, 340, 342, 346, 376, 380, 444, 446, 448, 450, 452, 455, 457, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 471, 472, 474, 475, 476, 479, 480, 481, 501, 502, 682, 683, 685, 694, 713, 714, 716, 719, 721, 724, 726, 729, 730, 731, 732, 733, 736, 738, 739, 744, 745, 746, 747, 748, 753, 754, 755, 756, 758, 759, 760, 762, 763, 764, 765, 767, 768, 769, 771, 773, 775, 776, 777, 778, 780, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 793, 794, 796, 798, 800, 801, 804, 808, 810, 811, 812, 814, 815, 817, 820, 828, 829, 830, 831, 832, 834, 835, 836, 840, 842, 844, 848, 850, 851, 852, 854, 857, 858, 859, 860, 861, 862, 863, 864, 865, 868, 873, 874, 875, 876, 877, 878, 880, 882, 885, 889, 890, 891, 892, 893, 894, 895, 898, 899, 900, 902, 905, 906, 907, 908, 909, 911, 915, 916, 918, 919, 921, 922, 924, 925, 927, 928, 929, 932, 933, 935, 938, 939, 941, 942, 944, 945, 946, 952, 956, 968, 1001, 1045, 1048, 1054, 1055, 1059, 1060, 1061, 1062, 1064, 1065, 1066, 1071, 1073, 1075, 1091, 1092, 1100, 1101, 1115, 1117, 1121, 1167, 1412, 1413, 1419, 1441, 1442, 1446, 1447, 1448, 1449, 1450, 1455, 1463, 1465, 1473, 1482, 1484, 1488, 1490, 1495, 1498, 1499, 1500, 1506, 1508, 1511, 1512, 1513, 1514, 1516, 1517, 1518, 1519, 1520, 1523, 1556, 1559, 1561, 1562, 1563, 1564, 1565, 1600, 4153, 4329, 23499, 40601, 40660, 40663, 40665, 40669, 40671, 40681, 40682, 40686, 40700, 40709, 40710, 40711, 40714, 40916]',\n",
       " 'Unknown origin': '[458, 1222, 1443, 1444, 1451, 1452, 1453, 1454]',\n",
       " 'Unspecified target feature': '[470, 679, 40687, 40869, 40976]'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = {}\n",
    "for key, value in sorted(data_status.items()):\n",
    "    v.setdefault(value, []).append(key)\n",
    "{k:str(v) for k,v in v.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Difference to google doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "names_selected = {v for k,v in data_names.items() if k in final_datasets}\n",
    "google_doc = {\n",
    "    \"phoneme\",\"breast-w\",\"Australian\",\"banknote-authentication\",\"eeg-eye-state\",\"electricity\",\n",
    "    \"analcatdata_dmft\",\"higgs\",\"blood-transfusion-service-center\",\"ilpd\",\"steel-plates-fault\",\n",
    "    \"satimage\",\"mfeat-morphological\",\"balance-scale\",\"credit-a\",\"cmc\",\"diabetes\",\"wilt\",\"MagicTelescope\",\n",
    "    \"vowel\",\"pc3\",\"adult\",\"pc4\",\"ada_agnostic\",\"GesturePhaseSegmentationProcessed\",\"PhishingWebsites\",\n",
    "    \"bank-marketing\",\"cardiotocography\",\"climate-model-simulation-crashes\",\"first-order-theorem-proving\",\n",
    "    \"wall-robot-navigation\",\"dresses-sales\",\"sick\",\"waveform-5000\",\"wdbc\",\"car\",\"tic-tac-toe\",\"mfeat-zernike\",\n",
    "    \"segment\",\"connect-4\",\"kc2\",\"jm1\",\"pc1\",\"kc1\",\"qsar-biodeg\",\"eucalyptus\",\"credit-g\",\"pendigits\",\"vehicle\",\n",
    "    \"letter\",\"optdigits\",\"mfeat-fourier\",\"mfeat-karhunen\",\"kr-vs-kp\",\"ozone-level-8hr\",\"sylva_agnostic\",\n",
    "    \"nomao\",\"spambase\",\"splice\",\"mushroom\",\"cylinder-bands\",\"SpeedDating\",\"texture\",\"mfeat-factors\",\"collins\",\n",
    "    \"mnist_784\",\"gina_agnostic\",\"Bioresponse\",\"Internet-Advertisements\",\"semeion\",\"soybean\",\"madelon\",\"har\",\n",
    "    \"isolet\",\"micro-mass\",\"cnae-9\",\"MiceProtein\",\"one-hundred-plants-margin\",\"one-hundred-plants-shape\",\n",
    "    \"one-hundred-plants-texture\",\"mfeat-pixel\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New ones!\n",
      "   Devnagari-Script\n",
      "   Fashion-MNIST\n",
      "   churn\n",
      "   credit-approval\n",
      "   dna\n",
      "   jungle_chess_2pcs_raw_endgame_complete\n",
      "   numerai28.6\n",
      "Dropped ones!\n",
      "   Internet-Advertisements\n",
      "   ada_agnostic\n",
      "   car\n",
      "   cardiotocography\n",
      "   credit-a\n",
      "   eeg-eye-state\n",
      "   gina_agnostic\n",
      "   mushroom\n",
      "   sylva_agnostic\n",
      "Difference\n",
      "   Devnagari-Script\n",
      "   Fashion-MNIST\n",
      "   Internet-Advertisements\n",
      "   ada_agnostic\n",
      "   car\n",
      "   cardiotocography\n",
      "   churn\n",
      "   credit-a\n",
      "   credit-approval\n",
      "   dna\n",
      "   eeg-eye-state\n",
      "   gina_agnostic\n",
      "   jungle_chess_2pcs_raw_endgame_complete\n",
      "   mushroom\n",
      "   numerai28.6\n",
      "   sylva_agnostic\n"
     ]
    }
   ],
   "source": [
    "print(\"New ones!\")\n",
    "for d in sorted(names_selected - google_doc):\n",
    "    print('  ', d)\n",
    "print(\"Dropped ones!\")\n",
    "for d in sorted(google_doc - names_selected):\n",
    "    print('  ', d)\n",
    "print('Difference')\n",
    "for d in sorted(google_doc ^ names_selected):\n",
    "    print('  ', d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}