data_wrangling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MoonBoard Data Wranglin'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Acquisition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File raw_probs.out already exists! Aborting to avoid an overwrite.\n",
      "# Lines, # Bytes:    2069 87184241\n"
     ]
    }
   ],
   "source": [
    "! ./data/download.sh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# With lines=True we can read the multi-json file all in one go. Each line is a different json dict,\n",
    "# but they all get bound together appropriately. (Each page is a row.)\n",
    "df = pd.read_json('data/raw_probs.out', lines=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check it out..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TknDts</th>\n",
       "      <th>Data</th>\n",
       "      <th>Total</th>\n",
       "      <th>AggregateResults</th>\n",
       "      <th>Errors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>5jIORLlYMuZTKkSY+Q4xhA3j6sT+CIY5ajfEEukkhxk=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'ACG3...</td>\n",
       "      <td>31034</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1252</th>\n",
       "      <td>XhLokMiZnwbDahkqfkins1J1s799GSaH+GcsBzSX8NM=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'VICT...</td>\n",
       "      <td>31034</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2003</th>\n",
       "      <td>HYA/Ebj2fJZnJhasFLX9ngLdrxn9ivymD7MfwmdATB0=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'DEAL...</td>\n",
       "      <td>31035</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>vAffQxJL+VGPP7NWqpYuEqX4eC5LPTtcXBfgayR/vgA=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'BEGI...</td>\n",
       "      <td>31034</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1340</th>\n",
       "      <td>HND5A8Kkk/0nLY6iRRN7vrbY2YJWBsQxh6OZ/XJNL28=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'RYAN...</td>\n",
       "      <td>31034</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>932</th>\n",
       "      <td>KFRJhdiYxinRrQyBYJ1/rSKKconye+yVpRdNwwnYgec=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'LIL ...</td>\n",
       "      <td>31034</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1751</th>\n",
       "      <td>agitagade9me8Ubg2Fodncvw+vcwFeGWCWifcoICsCo=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'BABA...</td>\n",
       "      <td>31035</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>H555rv8Vv4xHIlomGJA9g4urgrInI2mRjuAvZiy2HxM=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'RIGH...</td>\n",
       "      <td>31034</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>hsS6muc78DibeHsWG01QbXMQhDzQimqCv6SzAiOUBYI=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'ACG1...</td>\n",
       "      <td>31034</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1904</th>\n",
       "      <td>5yQgi8t63QMTs3RQQDNDRdBiQWU5bxCXgeWlopvH/i4=</td>\n",
       "      <td>[{'Method': 'Feet follow hands', 'Name': 'PORO...</td>\n",
       "      <td>31035</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            TknDts  \\\n",
       "47    5jIORLlYMuZTKkSY+Q4xhA3j6sT+CIY5ajfEEukkhxk=   \n",
       "1252  XhLokMiZnwbDahkqfkins1J1s799GSaH+GcsBzSX8NM=   \n",
       "2003  HYA/Ebj2fJZnJhasFLX9ngLdrxn9ivymD7MfwmdATB0=   \n",
       "200   vAffQxJL+VGPP7NWqpYuEqX4eC5LPTtcXBfgayR/vgA=   \n",
       "1340  HND5A8Kkk/0nLY6iRRN7vrbY2YJWBsQxh6OZ/XJNL28=   \n",
       "932   KFRJhdiYxinRrQyBYJ1/rSKKconye+yVpRdNwwnYgec=   \n",
       "1751  agitagade9me8Ubg2Fodncvw+vcwFeGWCWifcoICsCo=   \n",
       "79    H555rv8Vv4xHIlomGJA9g4urgrInI2mRjuAvZiy2HxM=   \n",
       "12    hsS6muc78DibeHsWG01QbXMQhDzQimqCv6SzAiOUBYI=   \n",
       "1904  5yQgi8t63QMTs3RQQDNDRdBiQWU5bxCXgeWlopvH/i4=   \n",
       "\n",
       "                                                   Data  Total  \\\n",
       "47    [{'Method': 'Feet follow hands', 'Name': 'ACG3...  31034   \n",
       "1252  [{'Method': 'Feet follow hands', 'Name': 'VICT...  31034   \n",
       "2003  [{'Method': 'Feet follow hands', 'Name': 'DEAL...  31035   \n",
       "200   [{'Method': 'Feet follow hands', 'Name': 'BEGI...  31034   \n",
       "1340  [{'Method': 'Feet follow hands', 'Name': 'RYAN...  31034   \n",
       "932   [{'Method': 'Feet follow hands', 'Name': 'LIL ...  31034   \n",
       "1751  [{'Method': 'Feet follow hands', 'Name': 'BABA...  31035   \n",
       "79    [{'Method': 'Feet follow hands', 'Name': 'RIGH...  31034   \n",
       "12    [{'Method': 'Feet follow hands', 'Name': 'ACG1...  31034   \n",
       "1904  [{'Method': 'Feet follow hands', 'Name': 'PORO...  31035   \n",
       "\n",
       "      AggregateResults  Errors  \n",
       "47                 NaN     NaN  \n",
       "1252               NaN     NaN  \n",
       "2003               NaN     NaN  \n",
       "200                NaN     NaN  \n",
       "1340               NaN     NaN  \n",
       "932                NaN     NaN  \n",
       "1751               NaN     NaN  \n",
       "79                 NaN     NaN  \n",
       "12                 NaN     NaN  \n",
       "1904               NaN     NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df.sample(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems like the only relevant data (to us) might be the 'Data' column. Yeah?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "# Are any of these not NaN?\n",
    "print((~df['Errors'].isnull()).any())\n",
    "print((~df['AggregateResults'].isnull()).any())\n",
    "# (Nope)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Yeah. And in fact 'Data' holds 15 problems each, since it represents the 15 problems of each \"page\" on the website."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Method': 'Feet follow hands',\n",
       "  'Name': 'PROBLEM 14',\n",
       "  'Grade': '7B',\n",
       "  'UserGrade': None,\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',\n",
       "   'Nickname': 'lucas Dourdourekas',\n",
       "   'Firstname': 'lucas',\n",
       "   'Lastname': 'Dourdourekas',\n",
       "   'City': 'Kalymnos',\n",
       "   'Country': 'GREECE ',\n",
       "   'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886636747434',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 3,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': False,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518093,\n",
       "    'Description': 'G4',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518094, 'Description': 'B7', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518095, 'Description': 'G8', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518096, 'Description': 'D11', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518097, 'Description': 'C13', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518098, 'Description': 'G17', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518099, 'Description': 'I18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 636,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 338,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 436,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 788,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 588,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 138,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '3 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'problem-14',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19228,\n",
       "  'ApiId': 19228,\n",
       "  'DateInserted': '/Date(1456826414000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'PROBLEM 13',\n",
       "  'Grade': '7A',\n",
       "  'UserGrade': '7A',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',\n",
       "   'Nickname': 'lucas Dourdourekas',\n",
       "   'Firstname': 'lucas',\n",
       "   'Lastname': 'Dourdourekas',\n",
       "   'City': 'Kalymnos',\n",
       "   'Country': 'GREECE ',\n",
       "   'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886636903684',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 43,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': False,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518087,\n",
       "    'Description': 'I4',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518088, 'Description': 'E7', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518089, 'Description': 'H11', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518090, 'Description': 'D12', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518091, 'Description': 'B16', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518092, 'Description': 'G18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 186,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 386,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 638,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 445,\n",
       "    'Y': 436,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 788,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '43 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'problem-13',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19227,\n",
       "  'ApiId': 19227,\n",
       "  'DateInserted': '/Date(1456826414000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'PROBLEM 12',\n",
       "  'Grade': '7B',\n",
       "  'UserGrade': None,\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',\n",
       "   'Nickname': 'lucas Dourdourekas',\n",
       "   'Firstname': 'lucas',\n",
       "   'Lastname': 'Dourdourekas',\n",
       "   'City': 'Kalymnos',\n",
       "   'Country': 'GREECE ',\n",
       "   'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886636903684',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 6,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': False,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518081,\n",
       "    'Description': 'G6',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518082, 'Description': 'K9', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518083, 'Description': 'H12', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518084, 'Description': 'G14', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518085, 'Description': 'B15', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518086, 'Description': 'A18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 95,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 236,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 688,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 288,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 445,\n",
       "    'Y': 386,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 595,\n",
       "    'Y': 538,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '6 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'problem-12',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19226,\n",
       "  'ApiId': 19226,\n",
       "  'DateInserted': '/Date(1456826415000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'PROBLEM 11',\n",
       "  'Grade': '7A',\n",
       "  'UserGrade': '7A',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',\n",
       "   'Nickname': 'lucas Dourdourekas',\n",
       "   'Firstname': 'lucas',\n",
       "   'Lastname': 'Dourdourekas',\n",
       "   'City': 'Kalymnos',\n",
       "   'Country': 'GREECE ',\n",
       "   'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886650044772',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 1576,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': True,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518074,\n",
       "    'Description': 'C5',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518075, 'Description': 'B6', 'IsStart': True, 'IsEnd': False},\n",
       "   {'Id': 1518076, 'Description': 'E8', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518077, 'Description': 'B10', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518078, 'Description': 'E14', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518079, 'Description': 'E16', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518080, 'Description': 'I18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 486,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 686,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 738,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 588,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 288,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 188,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '1576 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'problem-11',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19225,\n",
       "  'ApiId': 19225,\n",
       "  'DateInserted': '/Date(1456826414000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'TESS',\n",
       "  'Grade': '6C+',\n",
       "  'UserGrade': '6C+',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',\n",
       "   'Nickname': 'Ben Moon',\n",
       "   'Firstname': 'Ben',\n",
       "   'Lastname': 'Moon',\n",
       "   'City': 'Sheffield',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Users/Profile/5FC09F63-05F3-4DAE-A1A5-3AC22C37139A.jpeg?637253886692085371',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 4582,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': True,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518065,\n",
       "    'Description': 'F5',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518066, 'Description': 'I5', 'IsStart': True, 'IsEnd': False},\n",
       "   {'Id': 1518067, 'Description': 'F9', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518068, 'Description': 'C10', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518069, 'Description': 'F11', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518070, 'Description': 'C14', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518071, 'Description': 'C15', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518072, 'Description': 'E18', 'IsStart': False, 'IsEnd': True},\n",
       "   {'Id': 1518073, 'Description': 'E18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 488,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 288,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 238,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 436,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 536,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 736,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 738,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '4582 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'tess',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19224,\n",
       "  'ApiId': 19224,\n",
       "  'DateInserted': '/Date(1456826415000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'PROBLEM 9',\n",
       "  'Grade': '7A+',\n",
       "  'UserGrade': '7A+',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',\n",
       "   'Nickname': 'Ally Patrick',\n",
       "   'Firstname': 'Alistair',\n",
       "   'Lastname': 'Patrick',\n",
       "   'City': 'Manchester',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886695366613',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 1219,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': True,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518059,\n",
       "    'Description': 'J5',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518060, 'Description': 'H8', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518061, 'Description': 'C10', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518062, 'Description': 'E13', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518063, 'Description': 'B16', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518064, 'Description': 'G18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 186,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 488,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 338,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 445,\n",
       "    'Y': 586,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 545,\n",
       "    'Y': 736,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '1219 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'problem-9',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19223,\n",
       "  'ApiId': 19223,\n",
       "  'DateInserted': '/Date(1456826415000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'VURT',\n",
       "  'Grade': '7B',\n",
       "  'UserGrade': '7B',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',\n",
       "   'Nickname': 'Ben Moon',\n",
       "   'Firstname': 'Ben',\n",
       "   'Lastname': 'Moon',\n",
       "   'City': 'Sheffield',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Users/Profile/5FC09F63-05F3-4DAE-A1A5-3AC22C37139A.jpeg?637253886696616610',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 681,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': True,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518053,\n",
       "    'Description': 'B3',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518054, 'Description': 'E7', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518055, 'Description': 'C10', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518056, 'Description': 'F12', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518057, 'Description': 'G14', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518058, 'Description': 'I18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 836,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 488,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 638,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 386,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 288,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '681 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'vurt',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19222,\n",
       "  'ApiId': 19222,\n",
       "  'DateInserted': '/Date(1456826415000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'SHORTROUND',\n",
       "  'Grade': '6C+',\n",
       "  'UserGrade': '6C+',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',\n",
       "   'Nickname': 'Ally Patrick',\n",
       "   'Firstname': 'Alistair',\n",
       "   'Lastname': 'Patrick',\n",
       "   'City': 'Manchester',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886697241608',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 308,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': False,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518045,\n",
       "    'Description': 'D3',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518046, 'Description': 'B4', 'IsStart': True, 'IsEnd': False},\n",
       "   {'Id': 1518047, 'Description': 'C7', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518048, 'Description': 'F9', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518049, 'Description': 'C12', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518050, 'Description': 'D13', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518051, 'Description': 'A16', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518052, 'Description': 'C18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 95,\n",
       "    'Y': 188,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 786,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 638,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 388,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 336,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 836,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 536,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '308 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'shortround',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19221,\n",
       "  'ApiId': 19221,\n",
       "  'DateInserted': '/Date(1456826427000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'CRIME AND PUNISHMENT',\n",
       "  'Grade': '7B',\n",
       "  'UserGrade': '7B',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',\n",
       "   'Nickname': 'Ben Moon',\n",
       "   'Firstname': 'Ben',\n",
       "   'Lastname': 'Moon',\n",
       "   'City': 'Sheffield',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Users/Profile/5FC09F63-05F3-4DAE-A1A5-3AC22C37139A.jpeg?637253886697554113',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 2,\n",
       "  'Repeats': 225,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': False,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518038,\n",
       "    'Description': 'D3',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518039, 'Description': 'D6', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518040, 'Description': 'F9', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518041, 'Description': 'I12', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518042, 'Description': 'D13', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518043, 'Description': 'D16', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518044, 'Description': 'C18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 186,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 336,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 686,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 836,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 536,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 388,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '225 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'crime-and-punishment',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19220,\n",
       "  'ApiId': 19220,\n",
       "  'DateInserted': '/Date(1456826415000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'PROBLEM 5',\n",
       "  'Grade': '7A',\n",
       "  'UserGrade': '7A',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',\n",
       "   'Nickname': 'Ally Patrick',\n",
       "   'Firstname': 'Alistair',\n",
       "   'Lastname': 'Patrick',\n",
       "   'City': 'Manchester',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886697554113',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 94,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': False,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518030,\n",
       "    'Description': 'C5',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518031, 'Description': 'D7', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518032, 'Description': 'A9', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518033, 'Description': 'F11', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518034, 'Description': 'F13', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518035, 'Description': 'I14', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518036, 'Description': 'E16', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518037, 'Description': 'D18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 95,\n",
       "    'Y': 538,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 738,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 86,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 636,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 188,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 336,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 436,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 288,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '94 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'problem-5',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19219,\n",
       "  'ApiId': 19219,\n",
       "  'DateInserted': '/Date(1456826415000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'HARD TIMES',\n",
       "  'Grade': '7A',\n",
       "  'UserGrade': '7A',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',\n",
       "   'Nickname': 'Ben Moon',\n",
       "   'Firstname': 'Ben',\n",
       "   'Lastname': 'Moon',\n",
       "   'City': 'Sheffield',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Users/Profile/5FC09F63-05F3-4DAE-A1A5-3AC22C37139A.jpeg?637253886729638843',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 4445,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': True,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': 'Crimp',\n",
       "  'Moves': [{'Id': 1844764,\n",
       "    'Description': 'B10',\n",
       "    'IsStart': False,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1844765, 'Description': 'E6', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1844766, 'Description': 'E8', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1844767, 'Description': 'E12', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1844768, 'Description': 'E16', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1844769, 'Description': 'G4', 'IsStart': True, 'IsEnd': False},\n",
       "   {'Id': 1844770, 'Description': 'I18', 'IsStart': False, 'IsEnd': True},\n",
       "   {'Id': 1844771, 'Description': 'J5', 'IsStart': True, 'IsEnd': False}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 145,\n",
       "    'Y': 486,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 688,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 588,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 388,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 188,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 788,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 545,\n",
       "    'Y': 736,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '4445 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'hard-times',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19218,\n",
       "  'ApiId': 19218,\n",
       "  'DateInserted': '/Date(1456826415000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'PROBLEM 3',\n",
       "  'Grade': '6B+',\n",
       "  'UserGrade': '6B+',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',\n",
       "   'Nickname': 'Ally Patrick',\n",
       "   'Firstname': 'Alistair',\n",
       "   'Lastname': 'Patrick',\n",
       "   'City': 'Manchester',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886730107591',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 2,\n",
       "  'Repeats': 228,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': False,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518015,\n",
       "    'Description': 'F5',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518016, 'Description': 'J5', 'IsStart': True, 'IsEnd': False},\n",
       "   {'Id': 1518017, 'Description': 'F9', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518018, 'Description': 'C10', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518019, 'Description': 'G13', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518020, 'Description': 'E15', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518021, 'Description': 'I18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 488,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 238,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 536,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 736,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 338,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 545,\n",
       "    'Y': 736,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '228 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'problem-3',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19217,\n",
       "  'ApiId': 19217,\n",
       "  'DateInserted': '/Date(1456826416000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'WUTHERING HEIGHTS',\n",
       "  'Grade': '6B+',\n",
       "  'UserGrade': '6B+',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',\n",
       "   'Nickname': 'Ben Moon',\n",
       "   'Firstname': 'Ben',\n",
       "   'Lastname': 'Moon',\n",
       "   'City': 'Sheffield',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Users/Profile/5FC09F63-05F3-4DAE-A1A5-3AC22C37139A.jpeg?637253887028522195',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 13445,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': True,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518007,\n",
       "    'Description': 'F5',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518008, 'Description': 'J8', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518009, 'Description': 'I11', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518010, 'Description': 'K11', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518011, 'Description': 'G13', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518012, 'Description': 'E15', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518013, 'Description': 'E18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 238,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 88,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 736,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 395,\n",
       "    'Y': 338,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 495,\n",
       "    'Y': 438,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 545,\n",
       "    'Y': 586,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 595,\n",
       "    'Y': 438,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '13445 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'wuthering-heights',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19216,\n",
       "  'ApiId': 19216,\n",
       "  'DateInserted': '/Date(1456826427000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'},\n",
       " {'Method': 'Feet follow hands',\n",
       "  'Name': 'FAR FROM THE MADDING CROWD',\n",
       "  'Grade': '6B+',\n",
       "  'UserGrade': '6B+',\n",
       "  'MoonBoardConfiguration': None,\n",
       "  'MoonBoardConfigurationId': 0,\n",
       "  'Setter': {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',\n",
       "   'Nickname': 'Ben Moon',\n",
       "   'Firstname': 'Ben',\n",
       "   'Lastname': 'Moon',\n",
       "   'City': 'Sheffield',\n",
       "   'Country': 'United Kingdom',\n",
       "   'ProfileImageUrl': '/Content/Account/Users/Profile/5FC09F63-05F3-4DAE-A1A5-3AC22C37139A.jpeg?637253887175628990',\n",
       "   'CanShareData': True},\n",
       "  'FirstAscender': False,\n",
       "  'Rating': 0,\n",
       "  'UserRating': 3,\n",
       "  'Repeats': 10197,\n",
       "  'Attempts': 0,\n",
       "  'Holdsetup': {'Id': 1,\n",
       "   'Description': 'MoonBoard 2016',\n",
       "   'Setby': None,\n",
       "   'DateInserted': None,\n",
       "   'DateUpdated': None,\n",
       "   'DateDeleted': None,\n",
       "   'IsLocked': False,\n",
       "   'IsMini': False,\n",
       "   'Active': False,\n",
       "   'Holdsets': None,\n",
       "   'MoonBoardConfigurations': None,\n",
       "   'HoldLayoutId': 0,\n",
       "   'AllowClimbMethods': False},\n",
       "  'IsBenchmark': True,\n",
       "  'IsMaster': False,\n",
       "  'IsAssessmentProblem': False,\n",
       "  'ProblemType': None,\n",
       "  'Moves': [{'Id': 1518000,\n",
       "    'Description': 'E6',\n",
       "    'IsStart': True,\n",
       "    'IsEnd': False},\n",
       "   {'Id': 1518001, 'Description': 'C5', 'IsStart': True, 'IsEnd': False},\n",
       "   {'Id': 1518002, 'Description': 'E8', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518003, 'Description': 'F11', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518004, 'Description': 'C13', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518005, 'Description': 'D15', 'IsStart': False, 'IsEnd': False},\n",
       "   {'Id': 1518006, 'Description': 'D18', 'IsStart': False, 'IsEnd': True}],\n",
       "  'Holdsets': None,\n",
       "  'Locations': [{'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 738,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 195,\n",
       "    'Y': 338,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 86,\n",
       "    'Color': '0xFF0000',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 245,\n",
       "    'Y': 236,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 688,\n",
       "    'Color': '0x00FF00',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 295,\n",
       "    'Y': 588,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'},\n",
       "   {'Id': 0,\n",
       "    'Holdset': None,\n",
       "    'Description': None,\n",
       "    'X': 345,\n",
       "    'Y': 436,\n",
       "    'Color': '0x0000FF',\n",
       "    'Rotation': 0,\n",
       "    'Type': 0,\n",
       "    'HoldNumber': None,\n",
       "    'Direction': 0,\n",
       "    'DirectionString': 'N'}],\n",
       "  'RepeatText': '10197 climbers  have repeated this problem',\n",
       "  'NumberOfTries': None,\n",
       "  'NameForUrl': 'far-from-the-madding-crowd',\n",
       "  'Upgraded': False,\n",
       "  'Downgraded': False,\n",
       "  'Id': 19215,\n",
       "  'ApiId': 19215,\n",
       "  'DateInserted': '/Date(1456826427000)/',\n",
       "  'DateUpdated': None,\n",
       "  'DateDeleted': None,\n",
       "  'DateTimeString': '01 Mar 2016 10:00'}]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Name</th>\n",
       "      <th>Grade</th>\n",
       "      <th>UserGrade</th>\n",
       "      <th>MoonBoardConfiguration</th>\n",
       "      <th>MoonBoardConfigurationId</th>\n",
       "      <th>Setter</th>\n",
       "      <th>FirstAscender</th>\n",
       "      <th>Rating</th>\n",
       "      <th>UserRating</th>\n",
       "      <th>...</th>\n",
       "      <th>NumberOfTries</th>\n",
       "      <th>NameForUrl</th>\n",
       "      <th>Upgraded</th>\n",
       "      <th>Downgraded</th>\n",
       "      <th>Id</th>\n",
       "      <th>ApiId</th>\n",
       "      <th>DateInserted</th>\n",
       "      <th>DateUpdated</th>\n",
       "      <th>DateDeleted</th>\n",
       "      <th>DateTimeString</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>PROBLEM 14</td>\n",
       "      <td>7B</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>problem-14</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19228</td>\n",
       "      <td>19228</td>\n",
       "      <td>/Date(1456826414000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>PROBLEM 13</td>\n",
       "      <td>7A</td>\n",
       "      <td>7A</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>problem-13</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19227</td>\n",
       "      <td>19227</td>\n",
       "      <td>/Date(1456826414000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>PROBLEM 12</td>\n",
       "      <td>7B</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>problem-12</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19226</td>\n",
       "      <td>19226</td>\n",
       "      <td>/Date(1456826415000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>PROBLEM 11</td>\n",
       "      <td>7A</td>\n",
       "      <td>7A</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>problem-11</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19225</td>\n",
       "      <td>19225</td>\n",
       "      <td>/Date(1456826414000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>TESS</td>\n",
       "      <td>6C+</td>\n",
       "      <td>6C+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>tess</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19224</td>\n",
       "      <td>19224</td>\n",
       "      <td>/Date(1456826415000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>PROBLEM 9</td>\n",
       "      <td>7A+</td>\n",
       "      <td>7A+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>problem-9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19223</td>\n",
       "      <td>19223</td>\n",
       "      <td>/Date(1456826415000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>VURT</td>\n",
       "      <td>7B</td>\n",
       "      <td>7B</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>vurt</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19222</td>\n",
       "      <td>19222</td>\n",
       "      <td>/Date(1456826415000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>SHORTROUND</td>\n",
       "      <td>6C+</td>\n",
       "      <td>6C+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>shortround</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19221</td>\n",
       "      <td>19221</td>\n",
       "      <td>/Date(1456826427000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>CRIME AND PUNISHMENT</td>\n",
       "      <td>7B</td>\n",
       "      <td>7B</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>crime-and-punishment</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19220</td>\n",
       "      <td>19220</td>\n",
       "      <td>/Date(1456826415000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>PROBLEM 5</td>\n",
       "      <td>7A</td>\n",
       "      <td>7A</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>problem-5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19219</td>\n",
       "      <td>19219</td>\n",
       "      <td>/Date(1456826415000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>HARD TIMES</td>\n",
       "      <td>7A</td>\n",
       "      <td>7A</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>hard-times</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19218</td>\n",
       "      <td>19218</td>\n",
       "      <td>/Date(1456826415000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>PROBLEM 3</td>\n",
       "      <td>6B+</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>problem-3</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19217</td>\n",
       "      <td>19217</td>\n",
       "      <td>/Date(1456826416000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>WUTHERING HEIGHTS</td>\n",
       "      <td>6B+</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>wuthering-heights</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19216</td>\n",
       "      <td>19216</td>\n",
       "      <td>/Date(1456826427000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>FAR FROM THE MADDING CROWD</td>\n",
       "      <td>6B+</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>far-from-the-madding-crowd</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19215</td>\n",
       "      <td>19215</td>\n",
       "      <td>/Date(1456826427000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>01 Mar 2016 10:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Method                        Name Grade UserGrade  \\\n",
       "0   Feet follow hands                  PROBLEM 14    7B      None   \n",
       "1   Feet follow hands                  PROBLEM 13    7A        7A   \n",
       "2   Feet follow hands                  PROBLEM 12    7B      None   \n",
       "3   Feet follow hands                  PROBLEM 11    7A        7A   \n",
       "4   Feet follow hands                        TESS   6C+       6C+   \n",
       "5   Feet follow hands                   PROBLEM 9   7A+       7A+   \n",
       "6   Feet follow hands                        VURT    7B        7B   \n",
       "7   Feet follow hands                  SHORTROUND   6C+       6C+   \n",
       "8   Feet follow hands        CRIME AND PUNISHMENT    7B        7B   \n",
       "9   Feet follow hands                   PROBLEM 5    7A        7A   \n",
       "10  Feet follow hands                  HARD TIMES    7A        7A   \n",
       "11  Feet follow hands                   PROBLEM 3   6B+       6B+   \n",
       "12  Feet follow hands           WUTHERING HEIGHTS   6B+       6B+   \n",
       "13  Feet follow hands  FAR FROM THE MADDING CROWD   6B+       6B+   \n",
       "\n",
       "   MoonBoardConfiguration  MoonBoardConfigurationId  \\\n",
       "0                    None                         0   \n",
       "1                    None                         0   \n",
       "2                    None                         0   \n",
       "3                    None                         0   \n",
       "4                    None                         0   \n",
       "5                    None                         0   \n",
       "6                    None                         0   \n",
       "7                    None                         0   \n",
       "8                    None                         0   \n",
       "9                    None                         0   \n",
       "10                   None                         0   \n",
       "11                   None                         0   \n",
       "12                   None                         0   \n",
       "13                   None                         0   \n",
       "\n",
       "                                               Setter  FirstAscender  Rating  \\\n",
       "0   {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...          False       0   \n",
       "1   {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...          False       0   \n",
       "2   {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...          False       0   \n",
       "3   {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...          False       0   \n",
       "4   {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...          False       0   \n",
       "5   {'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',...          False       0   \n",
       "6   {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...          False       0   \n",
       "7   {'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',...          False       0   \n",
       "8   {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...          False       0   \n",
       "9   {'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',...          False       0   \n",
       "10  {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...          False       0   \n",
       "11  {'Id': 'CC7D7BD1-67CD-4027-A537-22E7855C18B7',...          False       0   \n",
       "12  {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...          False       0   \n",
       "13  {'Id': '5FC09F63-05F3-4DAE-A1A5-3AC22C37139A',...          False       0   \n",
       "\n",
       "    UserRating  ...  NumberOfTries                  NameForUrl Upgraded  \\\n",
       "0            3  ...           None                  problem-14    False   \n",
       "1            3  ...           None                  problem-13    False   \n",
       "2            3  ...           None                  problem-12    False   \n",
       "3            3  ...           None                  problem-11    False   \n",
       "4            3  ...           None                        tess    False   \n",
       "5            3  ...           None                   problem-9    False   \n",
       "6            3  ...           None                        vurt    False   \n",
       "7            3  ...           None                  shortround    False   \n",
       "8            2  ...           None        crime-and-punishment    False   \n",
       "9            3  ...           None                   problem-5    False   \n",
       "10           3  ...           None                  hard-times    False   \n",
       "11           2  ...           None                   problem-3    False   \n",
       "12           3  ...           None           wuthering-heights    False   \n",
       "13           3  ...           None  far-from-the-madding-crowd    False   \n",
       "\n",
       "    Downgraded     Id  ApiId           DateInserted DateUpdated DateDeleted  \\\n",
       "0        False  19228  19228  /Date(1456826414000)/        None        None   \n",
       "1        False  19227  19227  /Date(1456826414000)/        None        None   \n",
       "2        False  19226  19226  /Date(1456826415000)/        None        None   \n",
       "3        False  19225  19225  /Date(1456826414000)/        None        None   \n",
       "4        False  19224  19224  /Date(1456826415000)/        None        None   \n",
       "5        False  19223  19223  /Date(1456826415000)/        None        None   \n",
       "6        False  19222  19222  /Date(1456826415000)/        None        None   \n",
       "7        False  19221  19221  /Date(1456826427000)/        None        None   \n",
       "8        False  19220  19220  /Date(1456826415000)/        None        None   \n",
       "9        False  19219  19219  /Date(1456826415000)/        None        None   \n",
       "10       False  19218  19218  /Date(1456826415000)/        None        None   \n",
       "11       False  19217  19217  /Date(1456826416000)/        None        None   \n",
       "12       False  19216  19216  /Date(1456826427000)/        None        None   \n",
       "13       False  19215  19215  /Date(1456826427000)/        None        None   \n",
       "\n",
       "       DateTimeString  \n",
       "0   01 Mar 2016 10:00  \n",
       "1   01 Mar 2016 10:00  \n",
       "2   01 Mar 2016 10:00  \n",
       "3   01 Mar 2016 10:00  \n",
       "4   01 Mar 2016 10:00  \n",
       "5   01 Mar 2016 10:00  \n",
       "6   01 Mar 2016 10:00  \n",
       "7   01 Mar 2016 10:00  \n",
       "8   01 Mar 2016 10:00  \n",
       "9   01 Mar 2016 10:00  \n",
       "10  01 Mar 2016 10:00  \n",
       "11  01 Mar 2016 10:00  \n",
       "12  01 Mar 2016 10:00  \n",
       "13  01 Mar 2016 10:00  \n",
       "\n",
       "[14 rows x 31 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display(df['Data'][0])\n",
    "pd.DataFrame.from_records(df['Data'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cool, so from_records will accept a list of dicts. Let's concat all of df's 'Data' cells (each is a series) and then bring it back to a df."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Name</th>\n",
       "      <th>Grade</th>\n",
       "      <th>UserGrade</th>\n",
       "      <th>MoonBoardConfiguration</th>\n",
       "      <th>MoonBoardConfigurationId</th>\n",
       "      <th>Setter</th>\n",
       "      <th>FirstAscender</th>\n",
       "      <th>Rating</th>\n",
       "      <th>UserRating</th>\n",
       "      <th>...</th>\n",
       "      <th>NumberOfTries</th>\n",
       "      <th>NameForUrl</th>\n",
       "      <th>Upgraded</th>\n",
       "      <th>Downgraded</th>\n",
       "      <th>Id</th>\n",
       "      <th>ApiId</th>\n",
       "      <th>DateInserted</th>\n",
       "      <th>DateUpdated</th>\n",
       "      <th>DateDeleted</th>\n",
       "      <th>DateTimeString</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24274</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>CRIMP TSUNAMI</td>\n",
       "      <td>7B+</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '52335328-eb2c-4780-ac77-acbda91c4952',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>crimp-tsunami</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>348856</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1563913409983)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>23 Jul 2019 21:23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26565</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>LITTLE BUDDY</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '22ba739e-c12f-4bf6-a7c1-d422e7ce3f59',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>little-buddy</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>355132</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1572909124303)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>04 Nov 2019 23:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17033</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>NARDY V3</td>\n",
       "      <td>6B+</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'a2781fbc-7ffa-4d66-9dd6-f44fe8bdd385',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>nardy-v3</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>327445</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1538608883777)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>04 Oct 2018 00:21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14850</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>KALUNGA</td>\n",
       "      <td>7C+</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'e4575fea-44b7-4902-80fb-b19871e54be9',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>kalunga</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>320861</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1528482240433)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>08 Jun 2018 19:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6338</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>HEEL NAW</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'DE2319CF-EB4A-437C-B75E-EAB5CEF4B492',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>heel-naw</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>225152</td>\n",
       "      <td>225152</td>\n",
       "      <td>/Date(1499963656000)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>13 Jul 2017 17:34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30390</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>LEARN MORE AT HOME</td>\n",
       "      <td>7A</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '3c4dd719-de25-44e7-9472-768dc3b9121f',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>learn-more-at-home</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>366981</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1586096441930)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>05 Apr 2020 15:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11737</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>NO IDEA</td>\n",
       "      <td>7A+</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': '0410095f-b481-4194-9875-6b5d25aa6e1b',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>no-idea</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>308677</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1513044257073)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>12 Dec 2017 02:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12287</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>BOULDERYBOULDER</td>\n",
       "      <td>6C+</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'B59E0994-8394-48EC-AFDF-81987B2F2341',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>boulderyboulder</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>309916</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1513853599960)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>21 Dec 2017 10:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9495</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>RANDOM2</td>\n",
       "      <td>7A</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'F16093D1-3538-4072-9870-EBAA74E7C0EA',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>random2</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>305855</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1509160706527)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>28 Oct 2017 04:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22012</th>\n",
       "      <td>Feet follow hands</td>\n",
       "      <td>GRUBY NAPIĘTY 6A+</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{'Id': 'afac66f6-6fee-496b-bf59-23bdb1a06b2a',...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>gruby-napięty-6a</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>342302</td>\n",
       "      <td>0</td>\n",
       "      <td>/Date(1555607526173)/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>18 Apr 2019 18:12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Method                Name Grade UserGrade  \\\n",
       "24274  Feet follow hands       CRIMP TSUNAMI   7B+      None   \n",
       "26565  Feet follow hands        LITTLE BUDDY   6B+      None   \n",
       "17033  Feet follow hands            NARDY V3   6B+       6B+   \n",
       "14850  Feet follow hands             KALUNGA   7C+      None   \n",
       "6338   Feet follow hands            HEEL NAW   6B+      None   \n",
       "30390  Feet follow hands  LEARN MORE AT HOME    7A      None   \n",
       "11737  Feet follow hands             NO IDEA   7A+      None   \n",
       "12287  Feet follow hands     BOULDERYBOULDER   6C+      None   \n",
       "9495   Feet follow hands             RANDOM2    7A      None   \n",
       "22012  Feet follow hands   GRUBY NAPIĘTY 6A+   6B+      None   \n",
       "\n",
       "      MoonBoardConfiguration  MoonBoardConfigurationId  \\\n",
       "24274                   None                         0   \n",
       "26565                   None                         0   \n",
       "17033                   None                         0   \n",
       "14850                   None                         0   \n",
       "6338                    None                         0   \n",
       "30390                   None                         0   \n",
       "11737                   None                         0   \n",
       "12287                   None                         0   \n",
       "9495                    None                         0   \n",
       "22012                   None                         0   \n",
       "\n",
       "                                                  Setter  FirstAscender  \\\n",
       "24274  {'Id': '52335328-eb2c-4780-ac77-acbda91c4952',...          False   \n",
       "26565  {'Id': '22ba739e-c12f-4bf6-a7c1-d422e7ce3f59',...          False   \n",
       "17033  {'Id': 'a2781fbc-7ffa-4d66-9dd6-f44fe8bdd385',...          False   \n",
       "14850  {'Id': 'e4575fea-44b7-4902-80fb-b19871e54be9',...          False   \n",
       "6338   {'Id': 'DE2319CF-EB4A-437C-B75E-EAB5CEF4B492',...          False   \n",
       "30390  {'Id': '3c4dd719-de25-44e7-9472-768dc3b9121f',...          False   \n",
       "11737  {'Id': '0410095f-b481-4194-9875-6b5d25aa6e1b',...          False   \n",
       "12287  {'Id': 'B59E0994-8394-48EC-AFDF-81987B2F2341',...          False   \n",
       "9495   {'Id': 'F16093D1-3538-4072-9870-EBAA74E7C0EA',...          False   \n",
       "22012  {'Id': 'afac66f6-6fee-496b-bf59-23bdb1a06b2a',...          False   \n",
       "\n",
       "       Rating  UserRating  ...  NumberOfTries          NameForUrl Upgraded  \\\n",
       "24274       0           0  ...           None       crimp-tsunami    False   \n",
       "26565       0           2  ...           None        little-buddy    False   \n",
       "17033       0           3  ...           None            nardy-v3    False   \n",
       "14850       0           0  ...           None             kalunga    False   \n",
       "6338        0           3  ...           None            heel-naw    False   \n",
       "30390       0           0  ...           None  learn-more-at-home    False   \n",
       "11737       0           0  ...           None             no-idea    False   \n",
       "12287       0           0  ...           None     boulderyboulder    False   \n",
       "9495        0           0  ...           None             random2    False   \n",
       "22012       0           2  ...           None    gruby-napięty-6a    False   \n",
       "\n",
       "       Downgraded      Id   ApiId           DateInserted DateUpdated  \\\n",
       "24274       False  348856       0  /Date(1563913409983)/        None   \n",
       "26565       False  355132       0  /Date(1572909124303)/        None   \n",
       "17033       False  327445       0  /Date(1538608883777)/        None   \n",
       "14850       False  320861       0  /Date(1528482240433)/        None   \n",
       "6338        False  225152  225152  /Date(1499963656000)/        None   \n",
       "30390       False  366981       0  /Date(1586096441930)/        None   \n",
       "11737       False  308677       0  /Date(1513044257073)/        None   \n",
       "12287       False  309916       0  /Date(1513853599960)/        None   \n",
       "9495        False  305855       0  /Date(1509160706527)/        None   \n",
       "22012       False  342302       0  /Date(1555607526173)/        None   \n",
       "\n",
       "      DateDeleted     DateTimeString  \n",
       "24274        None  23 Jul 2019 21:23  \n",
       "26565        None  04 Nov 2019 23:12  \n",
       "17033        None  04 Oct 2018 00:21  \n",
       "14850        None  08 Jun 2018 19:24  \n",
       "6338         None  13 Jul 2017 17:34  \n",
       "30390        None  05 Apr 2020 15:20  \n",
       "11737        None  12 Dec 2017 02:04  \n",
       "12287        None  21 Dec 2017 10:53  \n",
       "9495         None  28 Oct 2017 04:18  \n",
       "22012        None  18 Apr 2019 18:12  \n",
       "\n",
       "[10 rows x 31 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Method                                                      Feet follow hands\n",
       "Name                                                               PROBLEM 14\n",
       "Grade                                                                      7B\n",
       "UserGrade                                                                None\n",
       "MoonBoardConfiguration                                                   None\n",
       "MoonBoardConfigurationId                                                    0\n",
       "Setter                      {'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13',...\n",
       "FirstAscender                                                           False\n",
       "Rating                                                                      0\n",
       "UserRating                                                                  3\n",
       "Repeats                                                                     3\n",
       "Attempts                                                                    0\n",
       "Holdsetup                   {'Id': 1, 'Description': 'MoonBoard 2016', 'Se...\n",
       "IsBenchmark                                                             False\n",
       "IsMaster                                                                False\n",
       "IsAssessmentProblem                                                     False\n",
       "ProblemType                                                              None\n",
       "Moves                       [{'Id': 1518093, 'Description': 'G4', 'IsStart...\n",
       "Holdsets                                                                 None\n",
       "Locations                   [{'Id': 0, 'Holdset': None, 'Description': Non...\n",
       "RepeatText                             3 climbers  have repeated this problem\n",
       "NumberOfTries                                                            None\n",
       "NameForUrl                                                         problem-14\n",
       "Upgraded                                                                False\n",
       "Downgraded                                                              False\n",
       "Id                                                                      19228\n",
       "ApiId                                                                   19228\n",
       "DateInserted                                            /Date(1456826414000)/\n",
       "DateUpdated                                                              None\n",
       "DateDeleted                                                              None\n",
       "DateTimeString                                              01 Mar 2016 10:00\n",
       "Name: 0, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_problem_dicts = sum([l for l in df['Data']], [])\n",
    "problems = pd.DataFrame.from_records(all_problem_dicts)\n",
    "display(problems.sample(10))\n",
    "display(problems.iloc[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From here we'll save all problem data for posterity before getting specifically to the columns that are likely relevant for us."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "problems.to_csv('data/raw_probs.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Id': '48B03D33-7FB0-43A9-BC1C-F9B75B2EFA13', 'Nickname': 'lucas Dourdourekas', 'Firstname': 'lucas', 'Lastname': 'Dourdourekas', 'City': 'Kalymnos', 'Country': 'GREECE ', 'ProfileImageUrl': '/Content/Account/Images/default-profile.png?637253886636747434', 'CanShareData': True}\n",
      "{'Id': 1, 'Description': 'MoonBoard 2016', 'Setby': None, 'DateInserted': None, 'DateUpdated': None, 'DateDeleted': None, 'IsLocked': False, 'IsMini': False, 'Active': False, 'Holdsets': None, 'MoonBoardConfigurations': None, 'HoldLayoutId': 0, 'AllowClimbMethods': False}\n",
      "[{'Id': 1518093, 'Description': 'G4', 'IsStart': True, 'IsEnd': False}, {'Id': 1518094, 'Description': 'B7', 'IsStart': False, 'IsEnd': False}, {'Id': 1518095, 'Description': 'G8', 'IsStart': False, 'IsEnd': False}, {'Id': 1518096, 'Description': 'D11', 'IsStart': False, 'IsEnd': False}, {'Id': 1518097, 'Description': 'C13', 'IsStart': False, 'IsEnd': False}, {'Id': 1518098, 'Description': 'G17', 'IsStart': False, 'IsEnd': False}, {'Id': 1518099, 'Description': 'I18', 'IsStart': False, 'IsEnd': True}]\n",
      "[{'Id': 0, 'Holdset': None, 'Description': None, 'X': 145, 'Y': 636, 'Color': '0x0000FF', 'Rotation': 0, 'Type': 0, 'HoldNumber': None, 'Direction': 0, 'DirectionString': 'N'}, {'Id': 0, 'Holdset': None, 'Description': None, 'X': 195, 'Y': 338, 'Color': '0x0000FF', 'Rotation': 0, 'Type': 0, 'HoldNumber': None, 'Direction': 0, 'DirectionString': 'N'}, {'Id': 0, 'Holdset': None, 'Description': None, 'X': 245, 'Y': 436, 'Color': '0x0000FF', 'Rotation': 0, 'Type': 0, 'HoldNumber': None, 'Direction': 0, 'DirectionString': 'N'}, {'Id': 0, 'Holdset': None, 'Description': None, 'X': 395, 'Y': 788, 'Color': '0x00FF00', 'Rotation': 0, 'Type': 0, 'HoldNumber': None, 'Direction': 0, 'DirectionString': 'N'}, {'Id': 0, 'Holdset': None, 'Description': None, 'X': 395, 'Y': 588, 'Color': '0x0000FF', 'Rotation': 0, 'Type': 0, 'HoldNumber': None, 'Direction': 0, 'DirectionString': 'N'}, {'Id': 0, 'Holdset': None, 'Description': None, 'X': 395, 'Y': 138, 'Color': '0x0000FF', 'Rotation': 0, 'Type': 0, 'HoldNumber': None, 'Direction': 0, 'DirectionString': 'N'}, {'Id': 0, 'Holdset': None, 'Description': None, 'X': 495, 'Y': 88, 'Color': '0xFF0000', 'Rotation': 0, 'Type': 0, 'HoldNumber': None, 'Direction': 0, 'DirectionString': 'N'}]\n"
     ]
    }
   ],
   "source": [
    "print(problems['Setter'][0])\n",
    "print(problems['Holdsetup'][0])\n",
    "print(problems['Moves'][0])\n",
    "print(problems['Locations'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n",
      "False\n",
      "False\n",
      "False\n",
      "False\n",
      "False\n",
      "False\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "print((problems['Method'] != 'Feet follow hands').any())\n",
    "print((~problems['MoonBoardConfiguration'].isnull()).any())\n",
    "print((problems['MoonBoardConfigurationId'] != 0).any())\n",
    "print((problems['Holdsetup'] != problems['Holdsetup'][0]).any())\n",
    "print(problems['Upgraded'].any())\n",
    "print(problems['Downgraded'].any())\n",
    "print((~problems['NumberOfTries'].isnull()).any())\n",
    "print((problems['Rating'] != 0).any())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "True\n",
      "True\n",
      "31034\n",
      "54\n"
     ]
    }
   ],
   "source": [
    "print((~problems['IsAssessmentProblem']).any())\n",
    "print((~problems['IsMaster']).any())\n",
    "print((~problems['IsBenchmark']).any())\n",
    "print(len(problems['Grade'] != problems['UserGrade']))\n",
    "print(len(problems.loc[~problems['ProblemType'].isnull()]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Of the deeper cells, it looks like Moves is the only really important one, and Setter is the problem setter's info. Many columns are uninformative (the same for the whole dataset). For some reason, 53 of these problems are of ProblemType 'Crimp.'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selection, formatting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Grade</th>\n",
       "      <th>UserGrade</th>\n",
       "      <th>Setter</th>\n",
       "      <th>UserRating</th>\n",
       "      <th>Repeats</th>\n",
       "      <th>IsBenchmark</th>\n",
       "      <th>IsMaster</th>\n",
       "      <th>IsAssessmentProblem</th>\n",
       "      <th>ProblemType</th>\n",
       "      <th>Moves</th>\n",
       "      <th>DateTimeString</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5201</th>\n",
       "      <td>P3</td>\n",
       "      <td>7C</td>\n",
       "      <td>None</td>\n",
       "      <td>{'Id': '641539DA-03D8-49A0-873C-3F4DC0EB7393',...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'Id': 1577932, 'Description': 'B4', 'IsStart...</td>\n",
       "      <td>20 May 2017 23:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29894</th>\n",
       "      <td>DIMEET V3</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>{'Id': '25bd5eb0-710c-4727-ba93-111357cc01b9',...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'Id': 2101219, 'Description': 'G18', 'IsStar...</td>\n",
       "      <td>12 Mar 2020 18:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19729</th>\n",
       "      <td>DER RETTER</td>\n",
       "      <td>6C+</td>\n",
       "      <td>None</td>\n",
       "      <td>{'Id': 'a65ba79c-5e2c-4573-94f6-b45637b2c670',...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'Id': 1865571, 'Description': 'K5', 'IsStart...</td>\n",
       "      <td>23 Jan 2019 13:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12918</th>\n",
       "      <td>BEAR DOWN MATT</td>\n",
       "      <td>6B+</td>\n",
       "      <td>6B+</td>\n",
       "      <td>{'Id': '1B5E4B63-91A0-4792-88AC-EEFFE4C027D4',...</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'Id': 1666860, 'Description': 'D5', 'IsStart...</td>\n",
       "      <td>03 Jan 2018 02:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11850</th>\n",
       "      <td>PJ POWER</td>\n",
       "      <td>7A</td>\n",
       "      <td>None</td>\n",
       "      <td>{'Id': 'F046F10A-7766-4949-8F25-492C2CB846E5',...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'Id': 1649212, 'Description': 'D10', 'IsStar...</td>\n",
       "      <td>13 Dec 2017 19:50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Name Grade UserGrade  \\\n",
       "5201               P3    7C      None   \n",
       "29894       DIMEET V3   6B+      None   \n",
       "19729      DER RETTER   6C+      None   \n",
       "12918  BEAR DOWN MATT   6B+       6B+   \n",
       "11850        PJ POWER    7A      None   \n",
       "\n",
       "                                                  Setter  UserRating  Repeats  \\\n",
       "5201   {'Id': '641539DA-03D8-49A0-873C-3F4DC0EB7393',...           3        1   \n",
       "29894  {'Id': '25bd5eb0-710c-4727-ba93-111357cc01b9',...           3        1   \n",
       "19729  {'Id': 'a65ba79c-5e2c-4573-94f6-b45637b2c670',...           3        1   \n",
       "12918  {'Id': '1B5E4B63-91A0-4792-88AC-EEFFE4C027D4',...           2        5   \n",
       "11850  {'Id': 'F046F10A-7766-4949-8F25-492C2CB846E5',...           0        0   \n",
       "\n",
       "       IsBenchmark  IsMaster  IsAssessmentProblem ProblemType  \\\n",
       "5201         False     False                False        None   \n",
       "29894        False     False                False        None   \n",
       "19729        False     False                False        None   \n",
       "12918        False     False                False        None   \n",
       "11850        False     False                False        None   \n",
       "\n",
       "                                                   Moves     DateTimeString  \n",
       "5201   [{'Id': 1577932, 'Description': 'B4', 'IsStart...  20 May 2017 23:06  \n",
       "29894  [{'Id': 2101219, 'Description': 'G18', 'IsStar...  12 Mar 2020 18:22  \n",
       "19729  [{'Id': 1865571, 'Description': 'K5', 'IsStart...  23 Jan 2019 13:19  \n",
       "12918  [{'Id': 1666860, 'Description': 'D5', 'IsStart...  03 Jan 2018 02:56  \n",
       "11850  [{'Id': 1649212, 'Description': 'D10', 'IsStar...  13 Dec 2017 19:50  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols2keep = ['Name', 'Grade', 'UserGrade', 'Setter', 'UserRating', 'Repeats', 'IsBenchmark',\n",
    "             'IsMaster', 'IsAssessmentProblem', 'ProblemType', 'Moves', 'DateTimeString']\n",
    "\n",
    "probs_clean = problems[cols2keep].copy()  # We do copy() because of column-adding operations next\n",
    "probs_clean.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only thing is that we want to expand on those deeper cells still (Moves and Setter)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Grade</th>\n",
       "      <th>UserGrade</th>\n",
       "      <th>UserRating</th>\n",
       "      <th>Repeats</th>\n",
       "      <th>IsBenchmark</th>\n",
       "      <th>IsMaster</th>\n",
       "      <th>IsAssessmentProblem</th>\n",
       "      <th>ProblemType</th>\n",
       "      <th>DateTimeString</th>\n",
       "      <th>Setter.Firstname</th>\n",
       "      <th>Setter.Lastname</th>\n",
       "      <th>Setter.Nickname</th>\n",
       "      <th>Setter.City</th>\n",
       "      <th>Setter.Country</th>\n",
       "      <th>Holds.Start</th>\n",
       "      <th>Holds.Intermed</th>\n",
       "      <th>Holds.End</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31029</th>\n",
       "      <td>24K GOLD</td>\n",
       "      <td>6C+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>17 May 2020 20:38</td>\n",
       "      <td>Tiffany</td>\n",
       "      <td>Wen</td>\n",
       "      <td>Tiff</td>\n",
       "      <td>Denver</td>\n",
       "      <td>USA</td>\n",
       "      <td>[I5]</td>\n",
       "      <td>[H9, D10, H14, C15, D16]</td>\n",
       "      <td>[C18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31030</th>\n",
       "      <td>14K GOLD</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>17 May 2020 20:36</td>\n",
       "      <td>Tiffany</td>\n",
       "      <td>Wen</td>\n",
       "      <td>Tiff</td>\n",
       "      <td>Denver</td>\n",
       "      <td>USA</td>\n",
       "      <td>[I5]</td>\n",
       "      <td>[H9, I12, H14, C15, D13, F9]</td>\n",
       "      <td>[E18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31031</th>\n",
       "      <td>CAREER LADDER</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>17 May 2020 14:35</td>\n",
       "      <td>Julian</td>\n",
       "      <td>Enzing</td>\n",
       "      <td>Enzko</td>\n",
       "      <td>Dresden</td>\n",
       "      <td>Germany</td>\n",
       "      <td>[I5]</td>\n",
       "      <td>[K5, G9, H10, G15, K14, I16]</td>\n",
       "      <td>[K18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31032</th>\n",
       "      <td>UP &amp; DOWN</td>\n",
       "      <td>6C+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>17 May 2020 10:55</td>\n",
       "      <td>Andrew</td>\n",
       "      <td>Aksenov</td>\n",
       "      <td>Andrew Aks</td>\n",
       "      <td>Tula</td>\n",
       "      <td>Russia</td>\n",
       "      <td>[H5, K5]</td>\n",
       "      <td>[J8, J9, G13, D12, B11, F6, C7, B16]</td>\n",
       "      <td>[D18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31033</th>\n",
       "      <td>BOB’S KABOB</td>\n",
       "      <td>7A</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>17 May 2020 03:54</td>\n",
       "      <td>Declan</td>\n",
       "      <td>Osgood</td>\n",
       "      <td>Declan Osgood</td>\n",
       "      <td>San Diego</td>\n",
       "      <td>United States</td>\n",
       "      <td>[B4, F6]</td>\n",
       "      <td>[K6, K9, K8, H12, G14, D16]</td>\n",
       "      <td>[D18]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Name Grade UserGrade  UserRating  Repeats  IsBenchmark  \\\n",
       "31029       24K GOLD   6C+      None           0        0        False   \n",
       "31030       14K GOLD   6B+      None           0        0        False   \n",
       "31031  CAREER LADDER   6B+      None           0        0        False   \n",
       "31032      UP & DOWN   6C+      None           0        0        False   \n",
       "31033    BOB’S KABOB    7A      None           0        0        False   \n",
       "\n",
       "       IsMaster  IsAssessmentProblem ProblemType     DateTimeString  \\\n",
       "31029     False                False        None  17 May 2020 20:38   \n",
       "31030     False                False        None  17 May 2020 20:36   \n",
       "31031     False                False        None  17 May 2020 14:35   \n",
       "31032     False                False        None  17 May 2020 10:55   \n",
       "31033     False                False        None  17 May 2020 03:54   \n",
       "\n",
       "      Setter.Firstname Setter.Lastname Setter.Nickname Setter.City  \\\n",
       "31029          Tiffany             Wen            Tiff      Denver   \n",
       "31030          Tiffany             Wen            Tiff      Denver   \n",
       "31031           Julian          Enzing           Enzko     Dresden   \n",
       "31032           Andrew         Aksenov      Andrew Aks        Tula   \n",
       "31033           Declan          Osgood   Declan Osgood  San Diego    \n",
       "\n",
       "      Setter.Country Holds.Start                        Holds.Intermed  \\\n",
       "31029            USA        [I5]              [H9, D10, H14, C15, D16]   \n",
       "31030            USA        [I5]          [H9, I12, H14, C15, D13, F9]   \n",
       "31031        Germany        [I5]          [K5, G9, H10, G15, K14, I16]   \n",
       "31032         Russia    [H5, K5]  [J8, J9, G13, D12, B11, F6, C7, B16]   \n",
       "31033  United States    [B4, F6]           [K6, K9, K8, H12, G14, D16]   \n",
       "\n",
       "      Holds.End  \n",
       "31029     [C18]  \n",
       "31030     [E18]  \n",
       "31031     [K18]  \n",
       "31032     [D18]  \n",
       "31033     [D18]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31034\n"
     ]
    }
   ],
   "source": [
    "# Expand on Setter info\n",
    "setters = probs_clean['Setter']\n",
    "keys = ['Firstname', 'Lastname', 'Nickname', 'City', 'Country']\n",
    "for k in keys:\n",
    "    vals = [s[k] for s in setters]\n",
    "    probs_clean['Setter.%s' % k] = vals\n",
    "\n",
    "# Expand on Moves info: filter moves by start, intermediate, end\n",
    "moves = probs_clean['Moves']\n",
    "def _moves2holdcols(dicts):\n",
    "    start, intermed, end = [], [], []\n",
    "    for d in dicts:\n",
    "        if d['IsStart']: start.append(d['Description'].upper())\n",
    "        elif d['IsEnd']: end.append(d['Description'].upper())\n",
    "        else: intermed.append(d['Description'].upper())\n",
    "    return [start, intermed, end]\n",
    "\n",
    "# Add columns for holds\n",
    "hold_cols = [_moves2holdcols(move_dicts) for move_dicts in moves]\n",
    "probs_clean['Holds.Start'] = [h[0] for h in hold_cols]\n",
    "probs_clean['Holds.Intermed'] = [h[1] for h in hold_cols]\n",
    "probs_clean['Holds.End'] = [h[2] for h in hold_cols]\n",
    "\n",
    "# Remove unexpanded cols\n",
    "probs_clean.drop('Setter', axis=1, inplace=True)\n",
    "probs_clean.drop('Moves', axis=1, inplace=True)\n",
    "\n",
    "display(probs_clean.tail())\n",
    "print(len(probs_clean))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deduplication\n",
    "\n",
    "Getting close! Things are falling into place, but we should check for dups.\n",
    "\n",
    "Interestingly, there are some some problems that are identical in terms of holds but differ in terms of name, and often grade. We don't prune those; just those that are obviously accidental dups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Grade</th>\n",
       "      <th>UserGrade</th>\n",
       "      <th>UserRating</th>\n",
       "      <th>Repeats</th>\n",
       "      <th>IsBenchmark</th>\n",
       "      <th>IsMaster</th>\n",
       "      <th>IsAssessmentProblem</th>\n",
       "      <th>ProblemType</th>\n",
       "      <th>DateTimeString</th>\n",
       "      <th>Setter.Firstname</th>\n",
       "      <th>Setter.Lastname</th>\n",
       "      <th>Setter.Nickname</th>\n",
       "      <th>Setter.City</th>\n",
       "      <th>Setter.Country</th>\n",
       "      <th>Holds.Start</th>\n",
       "      <th>Holds.Intermed</th>\n",
       "      <th>Holds.End</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5835</th>\n",
       "      <td>DAVIDS</td>\n",
       "      <td>7B</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>20 Jun 2017 15:30</td>\n",
       "      <td>Tommy</td>\n",
       "      <td>Skeide</td>\n",
       "      <td>Tommy Skeide</td>\n",
       "      <td>Ålesund</td>\n",
       "      <td>NORWAY</td>\n",
       "      <td>[A5]</td>\n",
       "      <td>[C8, E8, F12, C14]</td>\n",
       "      <td>[C18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5836</th>\n",
       "      <td>DAVIDS</td>\n",
       "      <td>7B</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>20 Jun 2017 15:30</td>\n",
       "      <td>Tommy</td>\n",
       "      <td>Skeide</td>\n",
       "      <td>Tommy Skeide</td>\n",
       "      <td>Ålesund</td>\n",
       "      <td>NORWAY</td>\n",
       "      <td>[A5]</td>\n",
       "      <td>[C8, E8, F12, C14]</td>\n",
       "      <td>[C18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3574</th>\n",
       "      <td>7A TOO MUCH</td>\n",
       "      <td>6C+</td>\n",
       "      <td>6C+</td>\n",
       "      <td>3</td>\n",
       "      <td>2214</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>11 Mar 2017 06:59</td>\n",
       "      <td>CH</td>\n",
       "      <td>Kuo</td>\n",
       "      <td>CH Kuo</td>\n",
       "      <td>taIpei</td>\n",
       "      <td>TAIWAN</td>\n",
       "      <td>[A5, C5]</td>\n",
       "      <td>[E8, E10, G15, I12, J16]</td>\n",
       "      <td>[E18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29513</th>\n",
       "      <td>DOUBLES #1-2</td>\n",
       "      <td>6B+</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>28 Feb 2020 01:07</td>\n",
       "      <td>Ben</td>\n",
       "      <td>Khounsombath</td>\n",
       "      <td>Ben Khounsombath</td>\n",
       "      <td>Charlotte</td>\n",
       "      <td>USA</td>\n",
       "      <td>[A5, C5]</td>\n",
       "      <td>[E8, E10, I12, G15, J16]</td>\n",
       "      <td>[E18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30702</th>\n",
       "      <td>JUST KEEP SWIMMING</td>\n",
       "      <td>6C</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>21 Apr 2020 20:28</td>\n",
       "      <td>Dawn</td>\n",
       "      <td>Brown</td>\n",
       "      <td>Dawn Brown</td>\n",
       "      <td>Doncaster</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>[A5, D5]</td>\n",
       "      <td>[D9, B11, B13, C16, F16]</td>\n",
       "      <td>[G18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24255</th>\n",
       "      <td>CRAWLER</td>\n",
       "      <td>6C+</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>23 Jul 2019 17:50</td>\n",
       "      <td>durukan</td>\n",
       "      <td>türe</td>\n",
       "      <td>ture</td>\n",
       "      <td>Mersin</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>[K5]</td>\n",
       "      <td>[J8, I11, H11, D12, D15, B16]</td>\n",
       "      <td>[E18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24254</th>\n",
       "      <td>MAMBA POISON</td>\n",
       "      <td>6C</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>23 Jul 2019 17:50</td>\n",
       "      <td>tolga han</td>\n",
       "      <td>edebal</td>\n",
       "      <td>yellowmamba</td>\n",
       "      <td>şanlıurfa</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>[K5]</td>\n",
       "      <td>[J8, I11, H11, D12, D15, B16]</td>\n",
       "      <td>[E18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3513</th>\n",
       "      <td>THUNDERMAN</td>\n",
       "      <td>6C</td>\n",
       "      <td>6C+</td>\n",
       "      <td>2</td>\n",
       "      <td>218</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>08 Mar 2017 03:07</td>\n",
       "      <td>Jack</td>\n",
       "      <td>McConnell</td>\n",
       "      <td>Jack McConnell</td>\n",
       "      <td>Colorado Springs</td>\n",
       "      <td>USA</td>\n",
       "      <td>[K5]</td>\n",
       "      <td>[J8, I10, H10, G13, E16]</td>\n",
       "      <td>[G18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16580</th>\n",
       "      <td>BEST KEPT SECRET</td>\n",
       "      <td>7A+</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>07 Sep 2018 22:59</td>\n",
       "      <td>Curtis</td>\n",
       "      <td>Labouff</td>\n",
       "      <td>Curtis Labouff</td>\n",
       "      <td>Las Vegas</td>\n",
       "      <td>UNITED STATES</td>\n",
       "      <td>[K5, D3]</td>\n",
       "      <td>[G9, F10, A12, B7, C15]</td>\n",
       "      <td>[A18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16579</th>\n",
       "      <td>BEST KEPT SECRET</td>\n",
       "      <td>7A+</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>07 Sep 2018 22:59</td>\n",
       "      <td>Curtis</td>\n",
       "      <td>Labouff</td>\n",
       "      <td>Curtis Labouff</td>\n",
       "      <td>Las Vegas</td>\n",
       "      <td>UNITED STATES</td>\n",
       "      <td>[K5, D3]</td>\n",
       "      <td>[G9, F10, A12, B7, C15]</td>\n",
       "      <td>[A18]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>129 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Name Grade UserGrade  UserRating  Repeats  IsBenchmark  \\\n",
       "5835               DAVIDS    7B      None           0        0        False   \n",
       "5836               DAVIDS    7B      None           3        2        False   \n",
       "3574          7A TOO MUCH   6C+       6C+           3     2214         True   \n",
       "29513        DOUBLES #1-2   6B+      None           2        2        False   \n",
       "30702  JUST KEEP SWIMMING    6C      None           0        0        False   \n",
       "...                   ...   ...       ...         ...      ...          ...   \n",
       "24255             CRAWLER   6C+      None           2       16        False   \n",
       "24254        MAMBA POISON    6C      None           3        8        False   \n",
       "3513           THUNDERMAN    6C       6C+           2      218        False   \n",
       "16580    BEST KEPT SECRET   7A+      None           3        1        False   \n",
       "16579    BEST KEPT SECRET   7A+      None           0        0        False   \n",
       "\n",
       "       IsMaster  IsAssessmentProblem ProblemType     DateTimeString  \\\n",
       "5835      False                False        None  20 Jun 2017 15:30   \n",
       "5836      False                False        None  20 Jun 2017 15:30   \n",
       "3574      False                False        None  11 Mar 2017 06:59   \n",
       "29513     False                False        None  28 Feb 2020 01:07   \n",
       "30702     False                False        None  21 Apr 2020 20:28   \n",
       "...         ...                  ...         ...                ...   \n",
       "24255     False                False        None  23 Jul 2019 17:50   \n",
       "24254     False                False        None  23 Jul 2019 17:50   \n",
       "3513      False                False        None  08 Mar 2017 03:07   \n",
       "16580     False                False        None  07 Sep 2018 22:59   \n",
       "16579     False                False        None  07 Sep 2018 22:59   \n",
       "\n",
       "      Setter.Firstname Setter.Lastname   Setter.Nickname        Setter.City  \\\n",
       "5835             Tommy          Skeide      Tommy Skeide            Ålesund   \n",
       "5836             Tommy          Skeide      Tommy Skeide            Ålesund   \n",
       "3574                CH             Kuo            CH Kuo             taIpei   \n",
       "29513              Ben    Khounsombath  Ben Khounsombath          Charlotte   \n",
       "30702             Dawn           Brown        Dawn Brown          Doncaster   \n",
       "...                ...             ...               ...                ...   \n",
       "24255         durukan             türe              ture             Mersin   \n",
       "24254        tolga han          edebal       yellowmamba          şanlıurfa   \n",
       "3513              Jack       McConnell    Jack McConnell  Colorado Springs    \n",
       "16580           Curtis         Labouff    Curtis Labouff          Las Vegas   \n",
       "16579           Curtis         Labouff    Curtis Labouff          Las Vegas   \n",
       "\n",
       "       Setter.Country Holds.Start                 Holds.Intermed Holds.End  \n",
       "5835           NORWAY        [A5]             [C8, E8, F12, C14]     [C18]  \n",
       "5836           NORWAY        [A5]             [C8, E8, F12, C14]     [C18]  \n",
       "3574           TAIWAN    [A5, C5]       [E8, E10, G15, I12, J16]     [E18]  \n",
       "29513             USA    [A5, C5]       [E8, E10, I12, G15, J16]     [E18]  \n",
       "30702  United Kingdom    [A5, D5]       [D9, B11, B13, C16, F16]     [G18]  \n",
       "...               ...         ...                            ...       ...  \n",
       "24255          Turkey        [K5]  [J8, I11, H11, D12, D15, B16]     [E18]  \n",
       "24254          Turkey        [K5]  [J8, I11, H11, D12, D15, B16]     [E18]  \n",
       "3513              USA        [K5]       [J8, I10, H10, G13, E16]     [G18]  \n",
       "16580   UNITED STATES    [K5, D3]        [G9, F10, A12, B7, C15]     [A18]  \n",
       "16579   UNITED STATES    [K5, D3]        [G9, F10, A12, B7, C15]     [A18]  \n",
       "\n",
       "[129 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Setter.Nickname</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2244</th>\n",
       "      <td>ACG70</td>\n",
       "      <td>bkkim</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2368</th>\n",
       "      <td>L'INSONNIA</td>\n",
       "      <td>antonio eremita</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2537</th>\n",
       "      <td>STILL DAVE</td>\n",
       "      <td>David Burian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4182</th>\n",
       "      <td>FORSØK 39</td>\n",
       "      <td>atoneni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5330</th>\n",
       "      <td>UH-000</td>\n",
       "      <td>ulises huete</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5441</th>\n",
       "      <td>35 GRADER</td>\n",
       "      <td>Tommy Skeide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5521</th>\n",
       "      <td>SUMMER CAMP V2</td>\n",
       "      <td>Matthew Smith</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5836</th>\n",
       "      <td>DAVIDS</td>\n",
       "      <td>Tommy Skeide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7396</th>\n",
       "      <td>CHRIS PROBLEM</td>\n",
       "      <td>Christopher Pattberg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8185</th>\n",
       "      <td>FRANK THE TANK</td>\n",
       "      <td>Sean Decaire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8463</th>\n",
       "      <td>DON’T TRY THAT CLIMB</td>\n",
       "      <td>Jack Davison</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9263</th>\n",
       "      <td>ВСЁ СНАЧАЛА№2</td>\n",
       "      <td>russian did it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9665</th>\n",
       "      <td>ЛЕГХКАЯ ПРОБЛЕМА</td>\n",
       "      <td>VASYA SKALOLAZKA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12558</th>\n",
       "      <td>SNIKERS</td>\n",
       "      <td>Luzan Matyas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13652</th>\n",
       "      <td>DANG MAGIC</td>\n",
       "      <td>Gus Carter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14208</th>\n",
       "      <td>YUMMI!</td>\n",
       "      <td>Leta Dickinson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14368</th>\n",
       "      <td>THE LAST BIG BANG</td>\n",
       "      <td>Gus Carter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14605</th>\n",
       "      <td>BZ - WALKING ON THE MOON</td>\n",
       "      <td>Thomas Watts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15258</th>\n",
       "      <td>DIONYSUS</td>\n",
       "      <td>Alexander Firmberger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15861</th>\n",
       "      <td>NOT MINE</td>\n",
       "      <td>IAmFrancoChan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15862</th>\n",
       "      <td>NOT MINE</td>\n",
       "      <td>IAmFrancoChan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15943</th>\n",
       "      <td>ELY</td>\n",
       "      <td>Taccheesvasi Taccheesvasi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16580</th>\n",
       "      <td>BEST KEPT SECRET</td>\n",
       "      <td>Curtis Labouff</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16648</th>\n",
       "      <td>RECEPÇÃO</td>\n",
       "      <td>Otavio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18001</th>\n",
       "      <td>SON OF KRYPTON</td>\n",
       "      <td>Chris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20565</th>\n",
       "      <td>MEG1</td>\n",
       "      <td>megoic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20887</th>\n",
       "      <td>JACK REACHER</td>\n",
       "      <td>Komjati Zoltan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21691</th>\n",
       "      <td>PIC A BANIC</td>\n",
       "      <td>Pseudosimian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23704</th>\n",
       "      <td>STARTERPROBLEM</td>\n",
       "      <td>vietan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24492</th>\n",
       "      <td>ÖKC WARMUP</td>\n",
       "      <td>Pitch13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26347</th>\n",
       "      <td>KHONSU</td>\n",
       "      <td>AvTheWiscoKid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28287</th>\n",
       "      <td>V4 LEMON SQUEEZEY</td>\n",
       "      <td>trooon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28644</th>\n",
       "      <td>AUSPROBIAT</td>\n",
       "      <td>TonyST</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29448</th>\n",
       "      <td>P.20</td>\n",
       "      <td>BAZIW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29449</th>\n",
       "      <td>P.20</td>\n",
       "      <td>BAZIW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29450</th>\n",
       "      <td>P.20</td>\n",
       "      <td>BAZIW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29452</th>\n",
       "      <td>P.19</td>\n",
       "      <td>BAZIW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29453</th>\n",
       "      <td>P.19</td>\n",
       "      <td>BAZIW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29454</th>\n",
       "      <td>P.19</td>\n",
       "      <td>BAZIW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29455</th>\n",
       "      <td>P.19</td>\n",
       "      <td>BAZIW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29456</th>\n",
       "      <td>P.19</td>\n",
       "      <td>BAZIW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30702</th>\n",
       "      <td>JUST KEEP SWIMMING</td>\n",
       "      <td>Dawn Brown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30867</th>\n",
       "      <td>WHATS SO FUNNY</td>\n",
       "      <td>Dawn Brown</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Name            Setter.Nickname\n",
       "2244                      ACG70                      bkkim\n",
       "2368                 L'INSONNIA            antonio eremita\n",
       "2537                 STILL DAVE               David Burian\n",
       "4182                  FORSØK 39                    atoneni\n",
       "5330                     UH-000               ulises huete\n",
       "5441                  35 GRADER               Tommy Skeide\n",
       "5521             SUMMER CAMP V2              Matthew Smith\n",
       "5836                     DAVIDS               Tommy Skeide\n",
       "7396              CHRIS PROBLEM       Christopher Pattberg\n",
       "8185             FRANK THE TANK               Sean Decaire\n",
       "8463       DON’T TRY THAT CLIMB               Jack Davison\n",
       "9263              ВСЁ СНАЧАЛА№2             russian did it\n",
       "9665           ЛЕГХКАЯ ПРОБЛЕМА           VASYA SKALOLAZKA\n",
       "12558                   SNIKERS               Luzan Matyas\n",
       "13652                DANG MAGIC                 Gus Carter\n",
       "14208                    YUMMI!             Leta Dickinson\n",
       "14368         THE LAST BIG BANG                 Gus Carter\n",
       "14605  BZ - WALKING ON THE MOON               Thomas Watts\n",
       "15258                  DIONYSUS       Alexander Firmberger\n",
       "15861                  NOT MINE              IAmFrancoChan\n",
       "15862                  NOT MINE              IAmFrancoChan\n",
       "15943                       ELY  Taccheesvasi Taccheesvasi\n",
       "16580          BEST KEPT SECRET             Curtis Labouff\n",
       "16648                  RECEPÇÃO                     Otavio\n",
       "18001            SON OF KRYPTON                      Chris\n",
       "20565                      MEG1                     megoic\n",
       "20887              JACK REACHER             Komjati Zoltan\n",
       "21691               PIC A BANIC               Pseudosimian\n",
       "23704            STARTERPROBLEM                     vietan\n",
       "24492                ÖKC WARMUP                    Pitch13\n",
       "26347                    KHONSU              AvTheWiscoKid\n",
       "28287         V4 LEMON SQUEEZEY                     trooon\n",
       "28644                AUSPROBIAT                     TonyST\n",
       "29448                      P.20                      BAZIW\n",
       "29449                      P.20                      BAZIW\n",
       "29450                      P.20                      BAZIW\n",
       "29452                      P.19                      BAZIW\n",
       "29453                      P.19                      BAZIW\n",
       "29454                      P.19                      BAZIW\n",
       "29455                      P.19                      BAZIW\n",
       "29456                      P.19                      BAZIW\n",
       "30702        JUST KEEP SWIMMING                 Dawn Brown\n",
       "30867            WHATS SO FUNNY                 Dawn Brown"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dups = probs_clean[probs_clean.applymap(lambda x: str(sorted(x)) if isinstance(x, list) else x).duplicated(['Holds.Start', 'Holds.Intermed', 'Holds.End'], keep=False)]\n",
    "display(dups.sort_values('Holds.Start'))\n",
    "\n",
    "names = dups[['Name', 'Setter.Nickname']]\n",
    "dup_names = names[names.duplicated()]\n",
    "display(dup_names)  # I'm not sure why some problems are still represented multiple times here; they shouldn't be"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "keeps, drops = set(), set()\n",
    "\n",
    "for _, (name, setter) in dup_names.iterrows():\n",
    "    # Get all probs that have the same name and setter\n",
    "    same_probs = dups[(dups['Name'] == name) & (dups['Setter.Nickname'] == setter)]\n",
    "    repeats, ratings = same_probs['Repeats'], same_probs['UserRating']\n",
    "    # Prefer higher repeats, then higher ratings, and otherwise newest\n",
    "    if (repeats.iloc[0] != repeats.iloc[1:]).any():\n",
    "        keep = repeats.idxmax()\n",
    "    elif (ratings.iloc[0] != ratings.iloc[1:]).any():\n",
    "        keep = ratings.idxmax()\n",
    "    else:\n",
    "        keep = same_probs.index[-1]\n",
    "    # Keep the preferred prob and drop the rest\n",
    "    keeps.add(keep)\n",
    "    drops.update(set(same_probs.index) - set([keep]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keeping 36 problems and dropping 43\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "30991"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Keeping {} problems and dropping {}'.format(len(keeps), len(drops)))\n",
    "\n",
    "probs_dedup = probs_clean.drop(drops).reset_index(drop=True)\n",
    "len(probs_dedup)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downgrade Problems\n",
    "\n",
    "Technically, the lowest grade in the MB2016 dataset is 6B+. However, some problems have names which suggest lower grades (6A+, 6A, ...). Let's change those.\n",
    "\n",
    "Interestingly, other problems which are also 6B+ have *higher* grades in their names. For these, however, it's unclear whether the setter was just being silly (and giving it an improper name) or lazy (and actually not changing the grade from 6B+). We don't do anything with these, even though it might be irksome, and there are a lot of 6B+ problems in this dataset.\n",
    "\n",
    "The other issue here is that some problems with the grade in their name could actually be referring to V-grade instead of Font grade. Unfortunately, we just don't have the information to make sure all problems are labeled correctly, We'll assume Font grade unless V-grade is implied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5681     EASY PEASY 5A\n",
       "8115     FÜR KOLLE. 5A\n",
       "16939          1985ABC\n",
       "17844     (5A)  30\" R.\n",
       "19354        PATACCA5A\n",
       "28245       CLUB_XD 5A\n",
       "Name: Name, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "20338                  SIMPLY 5B\n",
       "29529    ONE OF THE EASIEST (5B)\n",
       "Name: Name, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "16452         UFO TECHNOLOGY 5C\n",
       "17641                 MARVIN5C+\n",
       "27178                       5C+\n",
       "27885    KIDS WARMUP EASY 5C 6A\n",
       "Name: Name, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "4911     15DAND E DO THE CROSS!\n",
       "14162             BUTAMOCHI 25D\n",
       "14163                     25D 3\n",
       "Name: Name, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "5259       やまのくち  5+\n",
       "27792        5+ HARD\n",
       "27831        5+ EASY\n",
       "30224    YOUTH-2, 5+\n",
       "30225    YOUTH-1, 5+\n",
       "Name: Name, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "699       GORILLA BEGINS(6A/V3)\n",
       "871                          6A\n",
       "2179                    6A TEST\n",
       "2491                 THIS IS 6A\n",
       "2989                   GRADE 6A\n",
       "                  ...          \n",
       "27885    KIDS WARMUP EASY 5C 6A\n",
       "28128       SLOW AND STATIC(6A)\n",
       "28677            MIGHT BE 6AISH\n",
       "29284              EASY AF (6A)\n",
       "30223              YOUTH-3, 6A-\n",
       "Name: Name, Length: 72, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2559                        6A+\n",
       "3479              EASY WEST 6A+\n",
       "4193              6A+ EXPLISIVE\n",
       "4296               6A+? PARAISO\n",
       "5391         WHEN I GROW UP 6A+\n",
       "9363                 WARMUP 6A+\n",
       "9458            KAREN ROUTE 6A+\n",
       "12147    LÍNEA DIRECTA (6A+/6B)\n",
       "13057               BASE 10 6A+\n",
       "14145                 V3- (6A+)\n",
       "16454              NO(R)WAY 6A+\n",
       "17599          NOHAYPAMÁS (6A+)\n",
       "18112               GUARMAP 6A+\n",
       "18231                  PROJ 6A+\n",
       "18597              BOHLIVIA 6A+\n",
       "18599          ALLEZSSANDRA 6A+\n",
       "18603             BLACKBIRD 6A+\n",
       "19374                MARVIN 6A+\n",
       "19657           FOR GUNWOO(6A+)\n",
       "20566      MARKS PROBLEMS (6A+)\n",
       "21555             EASIEST (6A+)\n",
       "21984         GRUBY NAPIĘTY 6A+\n",
       "22302               PROBLEM 6A+\n",
       "22405                 FERMO 6A+\n",
       "23251                EASY E 6A+\n",
       "26804    WARM ME UP, SCOTTY 6A+\n",
       "27125                 AH. (6A+)\n",
       "Name: Name, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "3530                      KOBRA 6B\n",
       "3727                  ROOFTOOPS 6B\n",
       "4541                           6B?\n",
       "7836                       LÆTT 6B\n",
       "7946               ENDA LETTERE 6B\n",
       "9253                    6B WARM UP\n",
       "9711                  NICE WORK 6B\n",
       "10820      THE GREAT WALL 2.0 (6B)\n",
       "10821          THE GREAT WALL (6B)\n",
       "12147       LÍNEA DIRECTA (6A+/6B)\n",
       "13504             PROBABLY HARD 6B\n",
       "14108                       6BB <3\n",
       "15144    I THINK ABOUT 6B, MAYBE +\n",
       "16456       OUTLANDERS FEELINGS 6B\n",
       "17035        NOSOBESLASPRESAS 6B/+\n",
       "18349                 BALLETTO(6B)\n",
       "18600                  LUDOLAND 6B\n",
       "18601                 MARY XMAS 6B\n",
       "18716                    STRONG 6B\n",
       "19265                 CORATELLA 6B\n",
       "19373                     MARVIN6B\n",
       "19895           CARETTA CARETTA 6B\n",
       "20446                   FORSE (6B)\n",
       "21092         TINA BOOT CAMP 1- 6B\n",
       "21488                   6B THE 2ND\n",
       "21600             FLYING FEETS  6B\n",
       "22117                     EASY 6B?\n",
       "22247                       ABC 6B\n",
       "22274                       123 6B\n",
       "22275                    SECOND 6B\n",
       "22404                    PAKKIA 6B\n",
       "22576                   4 SHORT 6B\n",
       "22604                        6BBBB\n",
       "23107                     GEORGE6B\n",
       "26033                      SOFT 6B\n",
       "26652    CAN'T HOLD THE TOP(6B/6A)\n",
       "30400           RIGHT END PINCH 6B\n",
       "30659          SMASHING TASHA - 6B\n",
       "30751                WHITE GLUE 6B\n",
       "Name: Name, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "158\n"
     ]
    }
   ],
   "source": [
    "fives = ['5A', '5B', '5C', '5D', '5\\+']\n",
    "sixes = ['6A', '6A\\+', '6B']\n",
    "\n",
    "total = 0\n",
    "for font in fives:\n",
    "    candidates = probs_dedup['Name'].str.contains(font)\n",
    "    vs = probs_dedup[candidates]['Name'].str.contains('V5')\n",
    "    likely_fonts = probs_dedup[candidates & ~vs]\n",
    "    display(likely_fonts['Name'])\n",
    "    total += len(likely_fonts)\n",
    "    \n",
    "for font in sixes:\n",
    "    candidates = probs_dedup['Name'].str.contains(font)\n",
    "    vs = probs_dedup[candidates]['Name'].str.contains('V6')\n",
    "    if '+' in font:\n",
    "        likely_fonts = probs_dedup[candidates & ~vs]\n",
    "    else:\n",
    "        upgrades = probs_dedup[candidates]['Name'].str.contains(font + '\\+')\n",
    "        likely_fonts = probs_dedup[candidates & ~vs & ~upgrades]\n",
    "    display(likely_fonts['Name'])\n",
    "    total += len(likely_fonts)\n",
    "    \n",
    "print(total)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like these are all the problems we want to target for a grade change---158 of them! This is not a huge number compared to the whole dataset; however, it is somewhat of a huge addition to grossly unbalanced lower grades.\n",
    "\n",
    "I had worried that some problems might have grade-like strings in their names (but not actually be indicating a grade)... but it doesn't seem like that's an issue for us.\n",
    "\n",
    "There are some problems that list multiple grades: like \"5C 6A\" or what have you. Eh, we'll arbitrarily take the highest grade that the above search method detects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "probs_regrade = probs_dedup.copy()\n",
    "\n",
    "for font in fives:\n",
    "    candidates = probs_dedup['Name'].str.contains(font)\n",
    "    vs = probs_dedup[candidates]['Name'].str.contains('V5')\n",
    "    idxs = candidates & ~vs\n",
    "    # Regrade\n",
    "    probs_regrade.loc[idxs, 'Grade'] = '5+'\n",
    "    probs_regrade.loc[idxs, 'UserGrade'] = np.nan\n",
    "    \n",
    "for font in sixes:\n",
    "    candidates = probs_dedup['Name'].str.contains(font)\n",
    "    vs = probs_dedup[candidates]['Name'].str.contains('V6')\n",
    "    idxs = candidates & ~vs\n",
    "    if '+' not in font:\n",
    "        upgrades = probs_dedup[candidates]['Name'].str.contains(font + '\\+')\n",
    "        idxs = idxs & ~upgrades\n",
    "    # Regrade\n",
    "    probs_regrade.loc[idxs, 'Grade'] = font.replace(\"\\\\\", '')\n",
    "    probs_regrade.loc[idxs, 'UserGrade'] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Did that work? Answer: yep\n",
    "\n",
    "# for font in fives:\n",
    "#     candidates = probs_regrade['Name'].str.contains(font)\n",
    "#     vs = probs_regrade[candidates]['Name'].str.contains('V5')\n",
    "#     likely_fonts = probs_regrade[candidates & ~vs]\n",
    "#     display(likely_fonts[['Name', 'Grade']])\n",
    "    \n",
    "# for font in sixes:\n",
    "#     candidates = probs_regrade['Name'].str.contains(font)\n",
    "#     vs = probs_regrade[candidates]['Name'].str.contains('V6')\n",
    "#     if '+' in font:\n",
    "#         likely_fonts = probs_regrade[candidates & ~vs]\n",
    "#     else:\n",
    "#         upgrades = probs_regrade[candidates]['Name'].str.contains(font + '\\+')\n",
    "#         likely_fonts = probs_regrade[candidates & ~vs & ~upgrades]\n",
    "#     display(likely_fonts[['Name', 'Grade']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6B+    10118\n",
       "6C+     4685\n",
       "7A      4096\n",
       "6C      3902\n",
       "7A+     3097\n",
       "7B+     1637\n",
       "7B      1563\n",
       "7C      1060\n",
       "7C+      384\n",
       "8A       181\n",
       "6A        71\n",
       "8A+       51\n",
       "6B        48\n",
       "8B        29\n",
       "6A+       26\n",
       "8B+       24\n",
       "5+        19\n",
       "Name: Grade, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# One last look at this funky distribution:\n",
    "probs_regrade['Grade'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs_regrade.to_csv('data/cleaned_probs.csv', index=False)\n",
    "probs_regrade.to_json('data/cleaned_probs.json', orient='records')"
   ]
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3.8.2 64-bit ('venv': venv)",
   "language": "python",
   "name": "python38264bitvenvvenv62f2ef13a99a49ca88eefe3bb4a02605"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}