create_union_of_dataseets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Suspension Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>county_name</th>\n",
       "      <th>district_name</th>\n",
       "      <th>school_name</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>total_suspensions</th>\n",
       "      <th>unduplicated_students_suspended_total</th>\n",
       "      <th>unduplicated_students_suspended_defiance</th>\n",
       "      <th>suspension_rate_total</th>\n",
       "      <th>suspensions_violent_injury</th>\n",
       "      <th>suspensions_violent_no_injury</th>\n",
       "      <th>suspensions_weapons</th>\n",
       "      <th>suspensions_drugs</th>\n",
       "      <th>suspensions_defiance</th>\n",
       "      <th>suspensions_other</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>2233</td>\n",
       "      <td>44</td>\n",
       "      <td>38</td>\n",
       "      <td>3</td>\n",
       "      <td>1.7</td>\n",
       "      <td>14</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>2419</td>\n",
       "      <td>82</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>2.6</td>\n",
       "      <td>14</td>\n",
       "      <td>48</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>690</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>859</td>\n",
       "      <td>65</td>\n",
       "      <td>53</td>\n",
       "      <td>5</td>\n",
       "      <td>6.2</td>\n",
       "      <td>18</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>99</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183303</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>3646836</td>\n",
       "      <td>285036</td>\n",
       "      <td>169199</td>\n",
       "      <td>31565</td>\n",
       "      <td>4.6</td>\n",
       "      <td>36279</td>\n",
       "      <td>142864</td>\n",
       "      <td>9008</td>\n",
       "      <td>36552</td>\n",
       "      <td>48155</td>\n",
       "      <td>12178</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183304</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>4037882</td>\n",
       "      <td>300792</td>\n",
       "      <td>179640</td>\n",
       "      <td>33379</td>\n",
       "      <td>4.4</td>\n",
       "      <td>38786</td>\n",
       "      <td>150200</td>\n",
       "      <td>9588</td>\n",
       "      <td>38632</td>\n",
       "      <td>50668</td>\n",
       "      <td>12918</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183305</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>6384919</td>\n",
       "      <td>363406</td>\n",
       "      <td>223867</td>\n",
       "      <td>40218</td>\n",
       "      <td>3.5</td>\n",
       "      <td>46433</td>\n",
       "      <td>179219</td>\n",
       "      <td>11786</td>\n",
       "      <td>50547</td>\n",
       "      <td>59808</td>\n",
       "      <td>15613</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183306</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>5748545</td>\n",
       "      <td>343813</td>\n",
       "      <td>210691</td>\n",
       "      <td>38005</td>\n",
       "      <td>3.7</td>\n",
       "      <td>43394</td>\n",
       "      <td>169973</td>\n",
       "      <td>11070</td>\n",
       "      <td>47873</td>\n",
       "      <td>56789</td>\n",
       "      <td>14714</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183307</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>753611</td>\n",
       "      <td>19594</td>\n",
       "      <td>13712</td>\n",
       "      <td>2257</td>\n",
       "      <td>1.8</td>\n",
       "      <td>3039</td>\n",
       "      <td>9247</td>\n",
       "      <td>716</td>\n",
       "      <td>2674</td>\n",
       "      <td>3019</td>\n",
       "      <td>899</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>183308 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "0            2022-23               D          01         10017     0000000   \n",
       "1            2022-23               D          01         10017     0000000   \n",
       "2            2022-23               D          01         10017     0000000   \n",
       "3            2022-23               D          01         10017     0000000   \n",
       "4            2022-23               D          01         10017     0000000   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "183303       2017-18               T          00           NaN         NaN   \n",
       "183304       2017-18               T          00           NaN         NaN   \n",
       "183305       2017-18               T          00           NaN         NaN   \n",
       "183306       2017-18               T          00           NaN         NaN   \n",
       "183307       2017-18               T          00           NaN         NaN   \n",
       "\n",
       "       county_name                       district_name      school_name  \\\n",
       "0          Alameda  Alameda County Office of Education  District Office   \n",
       "1          Alameda  Alameda County Office of Education  District Office   \n",
       "2          Alameda  Alameda County Office of Education  District Office   \n",
       "3          Alameda  Alameda County Office of Education  District Office   \n",
       "4          Alameda  Alameda County Office of Education  District Office   \n",
       "...            ...                                 ...              ...   \n",
       "183303       State                                 NaN              NaN   \n",
       "183304       State                                 NaN              NaN   \n",
       "183305       State                                 NaN              NaN   \n",
       "183306       State                                 NaN              NaN   \n",
       "183307       State                                 NaN              NaN   \n",
       "\n",
       "       charter_yn reporting_category cumulative_enrollment total_suspensions  \\\n",
       "0             All                 GF                  2233                44   \n",
       "1             All                 GM                  2419                82   \n",
       "2             All                 RA                   690                 2   \n",
       "3             All                 RB                   859                65   \n",
       "4             All                 RD                    99                 3   \n",
       "...           ...                ...                   ...               ...   \n",
       "183303        No                  SS               3646836            285036   \n",
       "183304        All                 SS               4037882            300792   \n",
       "183305        All                 TA               6384919            363406   \n",
       "183306        No                  TA               5748545            343813   \n",
       "183307        Yes                 TA                753611             19594   \n",
       "\n",
       "       unduplicated_students_suspended_total  \\\n",
       "0                                         38   \n",
       "1                                         64   \n",
       "2                                          2   \n",
       "3                                         53   \n",
       "4                                          2   \n",
       "...                                      ...   \n",
       "183303                                169199   \n",
       "183304                                179640   \n",
       "183305                                223867   \n",
       "183306                                210691   \n",
       "183307                                 13712   \n",
       "\n",
       "       unduplicated_students_suspended_defiance suspension_rate_total  \\\n",
       "0                                             3                   1.7   \n",
       "1                                             6                   2.6   \n",
       "2                                             0                   0.3   \n",
       "3                                             5                   6.2   \n",
       "4                                             0                   2.0   \n",
       "...                                         ...                   ...   \n",
       "183303                                    31565                   4.6   \n",
       "183304                                    33379                   4.4   \n",
       "183305                                    40218                   3.5   \n",
       "183306                                    38005                   3.7   \n",
       "183307                                     2257                   1.8   \n",
       "\n",
       "       suspensions_violent_injury suspensions_violent_no_injury  \\\n",
       "0                              14                            20   \n",
       "1                              14                            48   \n",
       "2                               1                             1   \n",
       "3                              18                            37   \n",
       "4                               0                             2   \n",
       "...                           ...                           ...   \n",
       "183303                      36279                        142864   \n",
       "183304                      38786                        150200   \n",
       "183305                      46433                        179219   \n",
       "183306                      43394                        169973   \n",
       "183307                       3039                          9247   \n",
       "\n",
       "       suspensions_weapons suspensions_drugs suspensions_defiance  \\\n",
       "0                        0                 6                    3   \n",
       "1                        2                 7                    7   \n",
       "2                        0                 0                    0   \n",
       "3                        0                 4                    5   \n",
       "4                        0                 1                    0   \n",
       "...                    ...               ...                  ...   \n",
       "183303                9008             36552                48155   \n",
       "183304                9588             38632                50668   \n",
       "183305               11786             50547                59808   \n",
       "183306               11070             47873                56789   \n",
       "183307                 716              2674                 3019   \n",
       "\n",
       "       suspensions_other is_wcc_entity  \n",
       "0                      1         False  \n",
       "1                      4         False  \n",
       "2                      0         False  \n",
       "3                      1         False  \n",
       "4                      0         False  \n",
       "...                  ...           ...  \n",
       "183303             12178         False  \n",
       "183304             12918         False  \n",
       "183305             15613         False  \n",
       "183306             14714         False  \n",
       "183307               899         False  \n",
       "\n",
       "[183308 rows x 22 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suspension_df = pd.read_csv('final_long_datasets_domain/final_suspension_df.csv', encoding='ISO-8859-1',dtype=str)\n",
    "suspension_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>county_name</th>\n",
       "      <th>district_name</th>\n",
       "      <th>school_name</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>total_suspensions</th>\n",
       "      <th>unduplicated_students_suspended_total</th>\n",
       "      <th>unduplicated_students_suspended_defiance</th>\n",
       "      <th>suspension_rate_total</th>\n",
       "      <th>suspensions_violent_injury</th>\n",
       "      <th>suspensions_violent_no_injury</th>\n",
       "      <th>suspensions_weapons</th>\n",
       "      <th>suspensions_drugs</th>\n",
       "      <th>suspensions_defiance</th>\n",
       "      <th>suspensions_other</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>grade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>2233</td>\n",
       "      <td>44</td>\n",
       "      <td>38</td>\n",
       "      <td>3</td>\n",
       "      <td>0.01702</td>\n",
       "      <td>14</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>2419</td>\n",
       "      <td>82</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>0.02646</td>\n",
       "      <td>14</td>\n",
       "      <td>48</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>690</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00290</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>859</td>\n",
       "      <td>65</td>\n",
       "      <td>53</td>\n",
       "      <td>5</td>\n",
       "      <td>0.06170</td>\n",
       "      <td>18</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>99</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.02020</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183303</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>3646836</td>\n",
       "      <td>285036</td>\n",
       "      <td>169199</td>\n",
       "      <td>31565</td>\n",
       "      <td>0.04640</td>\n",
       "      <td>36279</td>\n",
       "      <td>142864</td>\n",
       "      <td>9008</td>\n",
       "      <td>36552</td>\n",
       "      <td>48155</td>\n",
       "      <td>12178</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183304</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>4037882</td>\n",
       "      <td>300792</td>\n",
       "      <td>179640</td>\n",
       "      <td>33379</td>\n",
       "      <td>0.04449</td>\n",
       "      <td>38786</td>\n",
       "      <td>150200</td>\n",
       "      <td>9588</td>\n",
       "      <td>38632</td>\n",
       "      <td>50668</td>\n",
       "      <td>12918</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183305</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>6384919</td>\n",
       "      <td>363406</td>\n",
       "      <td>223867</td>\n",
       "      <td>40218</td>\n",
       "      <td>0.03506</td>\n",
       "      <td>46433</td>\n",
       "      <td>179219</td>\n",
       "      <td>11786</td>\n",
       "      <td>50547</td>\n",
       "      <td>59808</td>\n",
       "      <td>15613</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183306</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>5748545</td>\n",
       "      <td>343813</td>\n",
       "      <td>210691</td>\n",
       "      <td>38005</td>\n",
       "      <td>0.03665</td>\n",
       "      <td>43394</td>\n",
       "      <td>169973</td>\n",
       "      <td>11070</td>\n",
       "      <td>47873</td>\n",
       "      <td>56789</td>\n",
       "      <td>14714</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183307</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>753611</td>\n",
       "      <td>19594</td>\n",
       "      <td>13712</td>\n",
       "      <td>2257</td>\n",
       "      <td>0.01820</td>\n",
       "      <td>3039</td>\n",
       "      <td>9247</td>\n",
       "      <td>716</td>\n",
       "      <td>2674</td>\n",
       "      <td>3019</td>\n",
       "      <td>899</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>183308 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "0            2022-23               D          01         10017     0000000   \n",
       "1            2022-23               D          01         10017     0000000   \n",
       "2            2022-23               D          01         10017     0000000   \n",
       "3            2022-23               D          01         10017     0000000   \n",
       "4            2022-23               D          01         10017     0000000   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "183303       2017-18               T          00         00000     0000000   \n",
       "183304       2017-18               T          00         00000     0000000   \n",
       "183305       2017-18               T          00         00000     0000000   \n",
       "183306       2017-18               T          00         00000     0000000   \n",
       "183307       2017-18               T          00         00000     0000000   \n",
       "\n",
       "       county_name                       district_name      school_name  \\\n",
       "0          Alameda  Alameda County Office of Education  District Office   \n",
       "1          Alameda  Alameda County Office of Education  District Office   \n",
       "2          Alameda  Alameda County Office of Education  District Office   \n",
       "3          Alameda  Alameda County Office of Education  District Office   \n",
       "4          Alameda  Alameda County Office of Education  District Office   \n",
       "...            ...                                 ...              ...   \n",
       "183303       State                                 NaN              NaN   \n",
       "183304       State                                 NaN              NaN   \n",
       "183305       State                                 NaN              NaN   \n",
       "183306       State                                 NaN              NaN   \n",
       "183307       State                                 NaN              NaN   \n",
       "\n",
       "       charter_yn reporting_category cumulative_enrollment total_suspensions  \\\n",
       "0             All                 GF                  2233                44   \n",
       "1             All                 GM                  2419                82   \n",
       "2             All                 RA                   690                 2   \n",
       "3             All                 RB                   859                65   \n",
       "4             All                 RD                    99                 3   \n",
       "...           ...                ...                   ...               ...   \n",
       "183303        No                  SS               3646836            285036   \n",
       "183304        All                 SS               4037882            300792   \n",
       "183305        All                 TA               6384919            363406   \n",
       "183306        No                  TA               5748545            343813   \n",
       "183307        Yes                 TA                753611             19594   \n",
       "\n",
       "       unduplicated_students_suspended_total  \\\n",
       "0                                         38   \n",
       "1                                         64   \n",
       "2                                          2   \n",
       "3                                         53   \n",
       "4                                          2   \n",
       "...                                      ...   \n",
       "183303                                169199   \n",
       "183304                                179640   \n",
       "183305                                223867   \n",
       "183306                                210691   \n",
       "183307                                 13712   \n",
       "\n",
       "       unduplicated_students_suspended_defiance  suspension_rate_total  \\\n",
       "0                                             3                0.01702   \n",
       "1                                             6                0.02646   \n",
       "2                                             0                0.00290   \n",
       "3                                             5                0.06170   \n",
       "4                                             0                0.02020   \n",
       "...                                         ...                    ...   \n",
       "183303                                    31565                0.04640   \n",
       "183304                                    33379                0.04449   \n",
       "183305                                    40218                0.03506   \n",
       "183306                                    38005                0.03665   \n",
       "183307                                     2257                0.01820   \n",
       "\n",
       "       suspensions_violent_injury suspensions_violent_no_injury  \\\n",
       "0                              14                            20   \n",
       "1                              14                            48   \n",
       "2                               1                             1   \n",
       "3                              18                            37   \n",
       "4                               0                             2   \n",
       "...                           ...                           ...   \n",
       "183303                      36279                        142864   \n",
       "183304                      38786                        150200   \n",
       "183305                      46433                        179219   \n",
       "183306                      43394                        169973   \n",
       "183307                       3039                          9247   \n",
       "\n",
       "       suspensions_weapons suspensions_drugs suspensions_defiance  \\\n",
       "0                        0                 6                    3   \n",
       "1                        2                 7                    7   \n",
       "2                        0                 0                    0   \n",
       "3                        0                 4                    5   \n",
       "4                        0                 1                    0   \n",
       "...                    ...               ...                  ...   \n",
       "183303                9008             36552                48155   \n",
       "183304                9588             38632                50668   \n",
       "183305               11786             50547                59808   \n",
       "183306               11070             47873                56789   \n",
       "183307                 716              2674                 3019   \n",
       "\n",
       "       suspensions_other is_wcc_entity  metric_id       grade  \n",
       "0                      1         False        601  All Grades  \n",
       "1                      4         False        601  All Grades  \n",
       "2                      0         False        601  All Grades  \n",
       "3                      1         False        601  All Grades  \n",
       "4                      0         False        601  All Grades  \n",
       "...                  ...           ...        ...         ...  \n",
       "183303             12178         False        601  All Grades  \n",
       "183304             12918         False        601  All Grades  \n",
       "183305             15613         False        601  All Grades  \n",
       "183306             14714         False        601  All Grades  \n",
       "183307               899         False        601  All Grades  \n",
       "\n",
       "[183308 rows x 24 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suspension_df.loc[suspension_df['aggregate_level'] == 'T', 'district_code'] = '00000'\n",
    "suspension_df.loc[suspension_df['aggregate_level'] == 'T', 'school_code'] = '0000000'\n",
    "suspension_df['metric_id'] = 601\n",
    "suspension_df['grade'] = 'All Grades'\n",
    "\n",
    "suspension_df['suspension_rate_total'] = (suspension_df['unduplicated_students_suspended_total'].astype(int) / suspension_df['cumulative_enrollment'].astype(int)).round(5)\n",
    "\n",
    "suspension_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>unduplicated_students_suspended_total</th>\n",
       "      <th>suspension_rate_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2233</td>\n",
       "      <td>38</td>\n",
       "      <td>0.01702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2419</td>\n",
       "      <td>64</td>\n",
       "      <td>0.02646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690</td>\n",
       "      <td>2</td>\n",
       "      <td>0.00290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>859</td>\n",
       "      <td>53</td>\n",
       "      <td>0.06170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>99</td>\n",
       "      <td>2</td>\n",
       "      <td>0.02020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183303</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3646836</td>\n",
       "      <td>169199</td>\n",
       "      <td>0.04640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183304</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>4037882</td>\n",
       "      <td>179640</td>\n",
       "      <td>0.04449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183305</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6384919</td>\n",
       "      <td>223867</td>\n",
       "      <td>0.03506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183306</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5748545</td>\n",
       "      <td>210691</td>\n",
       "      <td>0.03665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183307</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>753611</td>\n",
       "      <td>13712</td>\n",
       "      <td>0.01820</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>183308 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               D        601          01         10017   \n",
       "1            2022-23               D        601          01         10017   \n",
       "2            2022-23               D        601          01         10017   \n",
       "3            2022-23               D        601          01         10017   \n",
       "4            2022-23               D        601          01         10017   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "183303       2017-18               T        601          00         00000   \n",
       "183304       2017-18               T        601          00         00000   \n",
       "183305       2017-18               T        601          00         00000   \n",
       "183306       2017-18               T        601          00         00000   \n",
       "183307       2017-18               T        601          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  \\\n",
       "0          0000000        All                 GF  All Grades   \n",
       "1          0000000        All                 GM  All Grades   \n",
       "2          0000000        All                 RA  All Grades   \n",
       "3          0000000        All                 RB  All Grades   \n",
       "4          0000000        All                 RD  All Grades   \n",
       "...            ...        ...                ...         ...   \n",
       "183303     0000000        No                  SS  All Grades   \n",
       "183304     0000000        All                 SS  All Grades   \n",
       "183305     0000000        All                 TA  All Grades   \n",
       "183306     0000000        No                  TA  All Grades   \n",
       "183307     0000000        Yes                 TA  All Grades   \n",
       "\n",
       "       cumulative_enrollment unduplicated_students_suspended_total  \\\n",
       "0                       2233                                    38   \n",
       "1                       2419                                    64   \n",
       "2                        690                                     2   \n",
       "3                        859                                    53   \n",
       "4                         99                                     2   \n",
       "...                      ...                                   ...   \n",
       "183303               3646836                                169199   \n",
       "183304               4037882                                179640   \n",
       "183305               6384919                                223867   \n",
       "183306               5748545                                210691   \n",
       "183307                753611                                 13712   \n",
       "\n",
       "        suspension_rate_total  \n",
       "0                     0.01702  \n",
       "1                     0.02646  \n",
       "2                     0.00290  \n",
       "3                     0.06170  \n",
       "4                     0.02020  \n",
       "...                       ...  \n",
       "183303                0.04640  \n",
       "183304                0.04449  \n",
       "183305                0.03506  \n",
       "183306                0.03665  \n",
       "183307                0.01820  \n",
       "\n",
       "[183308 rows x 12 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the new column order\n",
    "new_order = [\n",
    "    'academic_year',\n",
    "    'aggregate_level',\n",
    "    'metric_id',\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'charter_yn',\n",
    "    'reporting_category',\n",
    "    'grade',\n",
    "    'cumulative_enrollment',\n",
    "    'unduplicated_students_suspended_total',\n",
    "    'suspension_rate_total'\n",
    "]\n",
    "suspension_df = suspension_df[new_order]\n",
    "suspension_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>total_all_cumulative_enrollment</th>\n",
       "      <th>total_all_unduplicated_students_suspended_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>4661</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>492</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>4187</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>61119</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>10690</td>\n",
       "      <td>290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>61119</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>9155</td>\n",
       "      <td>237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183252</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>S</td>\n",
       "      <td>07</td>\n",
       "      <td>61796</td>\n",
       "      <td>0732164</td>\n",
       "      <td>No</td>\n",
       "      <td>1458</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183253</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>S</td>\n",
       "      <td>07</td>\n",
       "      <td>61796</td>\n",
       "      <td>0730325</td>\n",
       "      <td>No</td>\n",
       "      <td>326</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183305</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>6384919</td>\n",
       "      <td>223867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183306</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>5748545</td>\n",
       "      <td>210691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183307</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>753611</td>\n",
       "      <td>13712</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14766 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "16           2022-23               D          01         10017     0000000   \n",
       "29           2022-23               D          01         10017     0000000   \n",
       "46           2022-23               D          01         10017     0000000   \n",
       "64           2022-23               D          01         61119     0000000   \n",
       "82           2022-23               D          01         61119     0000000   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "183252       2017-18               S          07         61796     0732164   \n",
       "183253       2017-18               S          07         61796     0730325   \n",
       "183305       2017-18               T          00         00000     0000000   \n",
       "183306       2017-18               T          00         00000     0000000   \n",
       "183307       2017-18               T          00         00000     0000000   \n",
       "\n",
       "       charter_yn total_all_cumulative_enrollment  \\\n",
       "16            All                            4661   \n",
       "29            No                              492   \n",
       "46            Yes                            4187   \n",
       "64            All                           10690   \n",
       "82            No                             9155   \n",
       "...           ...                             ...   \n",
       "183252        No                             1458   \n",
       "183253        No                              326   \n",
       "183305        All                         6384919   \n",
       "183306        No                          5748545   \n",
       "183307        Yes                          753611   \n",
       "\n",
       "       total_all_unduplicated_students_suspended_total  \n",
       "16                                                 102  \n",
       "29                                                   2  \n",
       "46                                                 100  \n",
       "64                                                 290  \n",
       "82                                                 237  \n",
       "...                                                ...  \n",
       "183252                                              78  \n",
       "183253                                               0  \n",
       "183305                                          223867  \n",
       "183306                                          210691  \n",
       "183307                                           13712  \n",
       "\n",
       "[14766 rows x 8 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_figures_df = suspension_df[suspension_df['reporting_category'] == 'TA'][['academic_year', 'aggregate_level', 'county_code', 'district_code', 'school_code', 'charter_yn', 'cumulative_enrollment', 'unduplicated_students_suspended_total']]\n",
    "\n",
    "total_figures_df = total_figures_df.rename(columns={\n",
    "    'cumulative_enrollment': 'total_all_cumulative_enrollment',\n",
    "    'unduplicated_students_suspended_total': 'total_all_unduplicated_students_suspended_total'  \n",
    "})\n",
    "\n",
    "total_figures_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "suspension_df = suspension_df.merge(total_figures_df, on=['academic_year', 'aggregate_level', 'county_code', 'district_code', 'school_code', 'charter_yn' ])\n",
    "suspension_df['cumulative_enrollment_diff'] = suspension_df['total_all_cumulative_enrollment'].astype(int) - suspension_df['cumulative_enrollment'].astype(int)\n",
    "suspension_df['unduplicated_students_suspended_total_diff'] = suspension_df['total_all_unduplicated_students_suspended_total'].astype(int) - suspension_df['unduplicated_students_suspended_total'].astype(int)\n",
    "\n",
    "suspension_df['suspension_rate_total_non_group'] = (suspension_df['unduplicated_students_suspended_total_diff'].astype(int) / suspension_df['cumulative_enrollment_diff'].astype(int) ).round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>unduplicated_students_suspended_total</th>\n",
       "      <th>suspension_rate_total</th>\n",
       "      <th>suspension_rate_total_non_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2233</td>\n",
       "      <td>38</td>\n",
       "      <td>0.01702</td>\n",
       "      <td>0.02636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2419</td>\n",
       "      <td>64</td>\n",
       "      <td>0.02646</td>\n",
       "      <td>0.01695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690</td>\n",
       "      <td>2</td>\n",
       "      <td>0.00290</td>\n",
       "      <td>0.02518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>859</td>\n",
       "      <td>53</td>\n",
       "      <td>0.06170</td>\n",
       "      <td>0.01289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>99</td>\n",
       "      <td>2</td>\n",
       "      <td>0.02020</td>\n",
       "      <td>0.02192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183303</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3646836</td>\n",
       "      <td>169199</td>\n",
       "      <td>0.04640</td>\n",
       "      <td>0.01974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183304</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>4037882</td>\n",
       "      <td>179640</td>\n",
       "      <td>0.04449</td>\n",
       "      <td>0.01884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183305</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6384919</td>\n",
       "      <td>223867</td>\n",
       "      <td>0.03506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183306</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5748545</td>\n",
       "      <td>210691</td>\n",
       "      <td>0.03665</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183307</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>753611</td>\n",
       "      <td>13712</td>\n",
       "      <td>0.01820</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>183308 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               D        601          01         10017   \n",
       "1            2022-23               D        601          01         10017   \n",
       "2            2022-23               D        601          01         10017   \n",
       "3            2022-23               D        601          01         10017   \n",
       "4            2022-23               D        601          01         10017   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "183303       2017-18               T        601          00         00000   \n",
       "183304       2017-18               T        601          00         00000   \n",
       "183305       2017-18               T        601          00         00000   \n",
       "183306       2017-18               T        601          00         00000   \n",
       "183307       2017-18               T        601          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  \\\n",
       "0          0000000        All                 GF  All Grades   \n",
       "1          0000000        All                 GM  All Grades   \n",
       "2          0000000        All                 RA  All Grades   \n",
       "3          0000000        All                 RB  All Grades   \n",
       "4          0000000        All                 RD  All Grades   \n",
       "...            ...        ...                ...         ...   \n",
       "183303     0000000        No                  SS  All Grades   \n",
       "183304     0000000        All                 SS  All Grades   \n",
       "183305     0000000        All                 TA  All Grades   \n",
       "183306     0000000        No                  TA  All Grades   \n",
       "183307     0000000        Yes                 TA  All Grades   \n",
       "\n",
       "       cumulative_enrollment unduplicated_students_suspended_total  \\\n",
       "0                       2233                                    38   \n",
       "1                       2419                                    64   \n",
       "2                        690                                     2   \n",
       "3                        859                                    53   \n",
       "4                         99                                     2   \n",
       "...                      ...                                   ...   \n",
       "183303               3646836                                169199   \n",
       "183304               4037882                                179640   \n",
       "183305               6384919                                223867   \n",
       "183306               5748545                                210691   \n",
       "183307                753611                                 13712   \n",
       "\n",
       "        suspension_rate_total  suspension_rate_total_non_group  \n",
       "0                     0.01702                          0.02636  \n",
       "1                     0.02646                          0.01695  \n",
       "2                     0.00290                          0.02518  \n",
       "3                     0.06170                          0.01289  \n",
       "4                     0.02020                          0.02192  \n",
       "...                       ...                              ...  \n",
       "183303                0.04640                          0.01974  \n",
       "183304                0.04449                          0.01884  \n",
       "183305                0.03506                              NaN  \n",
       "183306                0.03665                              NaN  \n",
       "183307                0.01820                              NaN  \n",
       "\n",
       "[183308 rows x 13 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the new column order\n",
    "new_order = [\n",
    "    'academic_year',\n",
    "    'aggregate_level',\n",
    "    'metric_id',\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'charter_yn',\n",
    "    'reporting_category',\n",
    "    'grade',\n",
    "    'cumulative_enrollment',\n",
    "    'unduplicated_students_suspended_total',\n",
    "    'suspension_rate_total',\n",
    "    'suspension_rate_total_non_group'\n",
    "]\n",
    "suspension_df = suspension_df[new_order]\n",
    "suspension_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Expulsion Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>county_name</th>\n",
       "      <th>district_name</th>\n",
       "      <th>school_name</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>total_expulsions</th>\n",
       "      <th>expelled_students_total</th>\n",
       "      <th>expelled_students_defiance_only</th>\n",
       "      <th>expulsion_rate_total</th>\n",
       "      <th>expulsion_count_violent_injury</th>\n",
       "      <th>expulsion_count_violent_no_injury</th>\n",
       "      <th>expulsion_count_weapons</th>\n",
       "      <th>expulsion_count_drugs</th>\n",
       "      <th>expulsion_count_defiance</th>\n",
       "      <th>expulsion_count_other</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>2233</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>2419</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>690</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>859</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>99</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191507</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>4037882</td>\n",
       "      <td>4578</td>\n",
       "      <td>4539</td>\n",
       "      <td>25</td>\n",
       "      <td>0.1</td>\n",
       "      <td>1078</td>\n",
       "      <td>1394</td>\n",
       "      <td>710</td>\n",
       "      <td>1241</td>\n",
       "      <td>26</td>\n",
       "      <td>129</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191508</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>478701</td>\n",
       "      <td>166</td>\n",
       "      <td>166</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28</td>\n",
       "      <td>54</td>\n",
       "      <td>25</td>\n",
       "      <td>52</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191509</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5369</td>\n",
       "      <td>5326</td>\n",
       "      <td>26</td>\n",
       "      <td>0.1</td>\n",
       "      <td>1207</td>\n",
       "      <td>1601</td>\n",
       "      <td>836</td>\n",
       "      <td>1545</td>\n",
       "      <td>27</td>\n",
       "      <td>153</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191510</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "      <td>189</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30</td>\n",
       "      <td>65</td>\n",
       "      <td>28</td>\n",
       "      <td>57</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191511</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>5748545</td>\n",
       "      <td>5180</td>\n",
       "      <td>5138</td>\n",
       "      <td>24</td>\n",
       "      <td>0.1</td>\n",
       "      <td>1177</td>\n",
       "      <td>1536</td>\n",
       "      <td>808</td>\n",
       "      <td>1488</td>\n",
       "      <td>25</td>\n",
       "      <td>146</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191512 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "0            2022-23               D          01         10017     0000000   \n",
       "1            2022-23               D          01         10017     0000000   \n",
       "2            2022-23               D          01         10017     0000000   \n",
       "3            2022-23               D          01         10017     0000000   \n",
       "4            2022-23               D          01         10017     0000000   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "191507       2017-18               T          00           NaN         NaN   \n",
       "191508       2017-18               T          00           NaN         NaN   \n",
       "191509       2017-18               T          00           NaN         NaN   \n",
       "191510       2017-18               T          00           NaN         NaN   \n",
       "191511       2017-18               T          00           NaN         NaN   \n",
       "\n",
       "       county_name                       district_name      school_name  \\\n",
       "0          Alameda  Alameda County Office of Education  District Office   \n",
       "1          Alameda  Alameda County Office of Education  District Office   \n",
       "2          Alameda  Alameda County Office of Education  District Office   \n",
       "3          Alameda  Alameda County Office of Education  District Office   \n",
       "4          Alameda  Alameda County Office of Education  District Office   \n",
       "...            ...                                 ...              ...   \n",
       "191507       State                                 NaN              NaN   \n",
       "191508       State                                 NaN              NaN   \n",
       "191509       State                                 NaN              NaN   \n",
       "191510       State                                 NaN              NaN   \n",
       "191511       State                                 NaN              NaN   \n",
       "\n",
       "       charter_yn reporting_category cumulative_enrollment total_expulsions  \\\n",
       "0             All                 GF                  2233                0   \n",
       "1             All                 GM                  2419                0   \n",
       "2             All                 RA                   690                0   \n",
       "3             All                 RB                   859                0   \n",
       "4             All                 RD                    99                0   \n",
       "...           ...                ...                   ...              ...   \n",
       "191507        All                 SS               4037882             4578   \n",
       "191508        Yes                 SS                478701              166   \n",
       "191509        All                 TA               6384919             5369   \n",
       "191510        Yes                 TA                753611              189   \n",
       "191511        No                  TA               5748545             5180   \n",
       "\n",
       "       expelled_students_total expelled_students_defiance_only  \\\n",
       "0                            0                               0   \n",
       "1                            0                               0   \n",
       "2                            0                               0   \n",
       "3                            0                               0   \n",
       "4                            0                               0   \n",
       "...                        ...                             ...   \n",
       "191507                    4539                              25   \n",
       "191508                     166                               2   \n",
       "191509                    5326                              26   \n",
       "191510                     189                               2   \n",
       "191511                    5138                              24   \n",
       "\n",
       "       expulsion_rate_total expulsion_count_violent_injury  \\\n",
       "0                       0.0                              0   \n",
       "1                       0.0                              0   \n",
       "2                       0.0                              0   \n",
       "3                       0.0                              0   \n",
       "4                       0.0                              0   \n",
       "...                     ...                            ...   \n",
       "191507                  0.1                           1078   \n",
       "191508                  0.0                             28   \n",
       "191509                  0.1                           1207   \n",
       "191510                  0.0                             30   \n",
       "191511                  0.1                           1177   \n",
       "\n",
       "       expulsion_count_violent_no_injury expulsion_count_weapons  \\\n",
       "0                                      0                       0   \n",
       "1                                      0                       0   \n",
       "2                                      0                       0   \n",
       "3                                      0                       0   \n",
       "4                                      0                       0   \n",
       "...                                  ...                     ...   \n",
       "191507                              1394                     710   \n",
       "191508                                54                      25   \n",
       "191509                              1601                     836   \n",
       "191510                                65                      28   \n",
       "191511                              1536                     808   \n",
       "\n",
       "       expulsion_count_drugs expulsion_count_defiance expulsion_count_other  \\\n",
       "0                          0                        0                     0   \n",
       "1                          0                        0                     0   \n",
       "2                          0                        0                     0   \n",
       "3                          0                        0                     0   \n",
       "4                          0                        0                     0   \n",
       "...                      ...                      ...                   ...   \n",
       "191507                  1241                       26                   129   \n",
       "191508                    52                        2                     5   \n",
       "191509                  1545                       27                   153   \n",
       "191510                    57                        2                     7   \n",
       "191511                  1488                       25                   146   \n",
       "\n",
       "       is_wcc_entity  \n",
       "0              False  \n",
       "1              False  \n",
       "2              False  \n",
       "3              False  \n",
       "4              False  \n",
       "...              ...  \n",
       "191507         False  \n",
       "191508         False  \n",
       "191509         False  \n",
       "191510         False  \n",
       "191511         False  \n",
       "\n",
       "[191512 rows x 22 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expulsion_df = pd.read_csv('final_long_datasets_domain/final_expulsion_df.csv', encoding='ISO-8859-1',dtype=str)\n",
    "\n",
    "expulsion_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>county_name</th>\n",
       "      <th>district_name</th>\n",
       "      <th>school_name</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>total_expulsions</th>\n",
       "      <th>expelled_students_total</th>\n",
       "      <th>expelled_students_defiance_only</th>\n",
       "      <th>expulsion_rate_total</th>\n",
       "      <th>expulsion_count_violent_injury</th>\n",
       "      <th>expulsion_count_violent_no_injury</th>\n",
       "      <th>expulsion_count_weapons</th>\n",
       "      <th>expulsion_count_drugs</th>\n",
       "      <th>expulsion_count_defiance</th>\n",
       "      <th>expulsion_count_other</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>grade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>2233</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>2419</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>690</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>859</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Alameda County Office of Education</td>\n",
       "      <td>District Office</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>99</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191507</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>4037882</td>\n",
       "      <td>4578</td>\n",
       "      <td>4539</td>\n",
       "      <td>25</td>\n",
       "      <td>0.00112</td>\n",
       "      <td>1078</td>\n",
       "      <td>1394</td>\n",
       "      <td>710</td>\n",
       "      <td>1241</td>\n",
       "      <td>26</td>\n",
       "      <td>129</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191508</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>478701</td>\n",
       "      <td>166</td>\n",
       "      <td>166</td>\n",
       "      <td>2</td>\n",
       "      <td>0.00035</td>\n",
       "      <td>28</td>\n",
       "      <td>54</td>\n",
       "      <td>25</td>\n",
       "      <td>52</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191509</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5369</td>\n",
       "      <td>5326</td>\n",
       "      <td>26</td>\n",
       "      <td>0.00083</td>\n",
       "      <td>1207</td>\n",
       "      <td>1601</td>\n",
       "      <td>836</td>\n",
       "      <td>1545</td>\n",
       "      <td>27</td>\n",
       "      <td>153</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191510</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "      <td>189</td>\n",
       "      <td>2</td>\n",
       "      <td>0.00025</td>\n",
       "      <td>30</td>\n",
       "      <td>65</td>\n",
       "      <td>28</td>\n",
       "      <td>57</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191511</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>5748545</td>\n",
       "      <td>5180</td>\n",
       "      <td>5138</td>\n",
       "      <td>24</td>\n",
       "      <td>0.00089</td>\n",
       "      <td>1177</td>\n",
       "      <td>1536</td>\n",
       "      <td>808</td>\n",
       "      <td>1488</td>\n",
       "      <td>25</td>\n",
       "      <td>146</td>\n",
       "      <td>False</td>\n",
       "      <td>602</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191512 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "0            2022-23               D          01         10017     0000000   \n",
       "1            2022-23               D          01         10017     0000000   \n",
       "2            2022-23               D          01         10017     0000000   \n",
       "3            2022-23               D          01         10017     0000000   \n",
       "4            2022-23               D          01         10017     0000000   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "191507       2017-18               T          00         00000     0000000   \n",
       "191508       2017-18               T          00         00000     0000000   \n",
       "191509       2017-18               T          00         00000     0000000   \n",
       "191510       2017-18               T          00         00000     0000000   \n",
       "191511       2017-18               T          00         00000     0000000   \n",
       "\n",
       "       county_name                       district_name      school_name  \\\n",
       "0          Alameda  Alameda County Office of Education  District Office   \n",
       "1          Alameda  Alameda County Office of Education  District Office   \n",
       "2          Alameda  Alameda County Office of Education  District Office   \n",
       "3          Alameda  Alameda County Office of Education  District Office   \n",
       "4          Alameda  Alameda County Office of Education  District Office   \n",
       "...            ...                                 ...              ...   \n",
       "191507       State                                 NaN              NaN   \n",
       "191508       State                                 NaN              NaN   \n",
       "191509       State                                 NaN              NaN   \n",
       "191510       State                                 NaN              NaN   \n",
       "191511       State                                 NaN              NaN   \n",
       "\n",
       "       charter_yn reporting_category cumulative_enrollment total_expulsions  \\\n",
       "0             All                 GF                  2233                0   \n",
       "1             All                 GM                  2419                0   \n",
       "2             All                 RA                   690                0   \n",
       "3             All                 RB                   859                0   \n",
       "4             All                 RD                    99                0   \n",
       "...           ...                ...                   ...              ...   \n",
       "191507        All                 SS               4037882             4578   \n",
       "191508        Yes                 SS                478701              166   \n",
       "191509        All                 TA               6384919             5369   \n",
       "191510        Yes                 TA                753611              189   \n",
       "191511        No                  TA               5748545             5180   \n",
       "\n",
       "       expelled_students_total expelled_students_defiance_only  \\\n",
       "0                            0                               0   \n",
       "1                            0                               0   \n",
       "2                            0                               0   \n",
       "3                            0                               0   \n",
       "4                            0                               0   \n",
       "...                        ...                             ...   \n",
       "191507                    4539                              25   \n",
       "191508                     166                               2   \n",
       "191509                    5326                              26   \n",
       "191510                     189                               2   \n",
       "191511                    5138                              24   \n",
       "\n",
       "        expulsion_rate_total expulsion_count_violent_injury  \\\n",
       "0                    0.00000                              0   \n",
       "1                    0.00000                              0   \n",
       "2                    0.00000                              0   \n",
       "3                    0.00000                              0   \n",
       "4                    0.00000                              0   \n",
       "...                      ...                            ...   \n",
       "191507               0.00112                           1078   \n",
       "191508               0.00035                             28   \n",
       "191509               0.00083                           1207   \n",
       "191510               0.00025                             30   \n",
       "191511               0.00089                           1177   \n",
       "\n",
       "       expulsion_count_violent_no_injury expulsion_count_weapons  \\\n",
       "0                                      0                       0   \n",
       "1                                      0                       0   \n",
       "2                                      0                       0   \n",
       "3                                      0                       0   \n",
       "4                                      0                       0   \n",
       "...                                  ...                     ...   \n",
       "191507                              1394                     710   \n",
       "191508                                54                      25   \n",
       "191509                              1601                     836   \n",
       "191510                                65                      28   \n",
       "191511                              1536                     808   \n",
       "\n",
       "       expulsion_count_drugs expulsion_count_defiance expulsion_count_other  \\\n",
       "0                          0                        0                     0   \n",
       "1                          0                        0                     0   \n",
       "2                          0                        0                     0   \n",
       "3                          0                        0                     0   \n",
       "4                          0                        0                     0   \n",
       "...                      ...                      ...                   ...   \n",
       "191507                  1241                       26                   129   \n",
       "191508                    52                        2                     5   \n",
       "191509                  1545                       27                   153   \n",
       "191510                    57                        2                     7   \n",
       "191511                  1488                       25                   146   \n",
       "\n",
       "       is_wcc_entity  metric_id       grade  \n",
       "0              False        602  All Grades  \n",
       "1              False        602  All Grades  \n",
       "2              False        602  All Grades  \n",
       "3              False        602  All Grades  \n",
       "4              False        602  All Grades  \n",
       "...              ...        ...         ...  \n",
       "191507         False        602  All Grades  \n",
       "191508         False        602  All Grades  \n",
       "191509         False        602  All Grades  \n",
       "191510         False        602  All Grades  \n",
       "191511         False        602  All Grades  \n",
       "\n",
       "[191512 rows x 24 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expulsion_df.loc[expulsion_df['aggregate_level'] == 'T', 'district_code'] = '00000'\n",
    "expulsion_df.loc[expulsion_df['aggregate_level'] == 'T', 'school_code'] = '0000000'\n",
    "expulsion_df['metric_id'] = 602\n",
    "expulsion_df['grade'] = 'All Grades'\n",
    "\n",
    "expulsion_df['expulsion_rate_total'] = (expulsion_df['expelled_students_total'].astype(int) / expulsion_df['cumulative_enrollment'].astype(int)).round(5)\n",
    "expulsion_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>expelled_students_total</th>\n",
       "      <th>expulsion_rate_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2233</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2419</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>859</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>99</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191507</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>4037882</td>\n",
       "      <td>4539</td>\n",
       "      <td>0.00112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191508</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>478701</td>\n",
       "      <td>166</td>\n",
       "      <td>0.00035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191509</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5326</td>\n",
       "      <td>0.00083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191510</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "      <td>0.00025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191511</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5748545</td>\n",
       "      <td>5138</td>\n",
       "      <td>0.00089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191512 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               D        602          01         10017   \n",
       "1            2022-23               D        602          01         10017   \n",
       "2            2022-23               D        602          01         10017   \n",
       "3            2022-23               D        602          01         10017   \n",
       "4            2022-23               D        602          01         10017   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "191507       2017-18               T        602          00         00000   \n",
       "191508       2017-18               T        602          00         00000   \n",
       "191509       2017-18               T        602          00         00000   \n",
       "191510       2017-18               T        602          00         00000   \n",
       "191511       2017-18               T        602          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  \\\n",
       "0          0000000        All                 GF  All Grades   \n",
       "1          0000000        All                 GM  All Grades   \n",
       "2          0000000        All                 RA  All Grades   \n",
       "3          0000000        All                 RB  All Grades   \n",
       "4          0000000        All                 RD  All Grades   \n",
       "...            ...        ...                ...         ...   \n",
       "191507     0000000        All                 SS  All Grades   \n",
       "191508     0000000        Yes                 SS  All Grades   \n",
       "191509     0000000        All                 TA  All Grades   \n",
       "191510     0000000        Yes                 TA  All Grades   \n",
       "191511     0000000        No                  TA  All Grades   \n",
       "\n",
       "       cumulative_enrollment expelled_students_total  expulsion_rate_total  \n",
       "0                       2233                       0               0.00000  \n",
       "1                       2419                       0               0.00000  \n",
       "2                        690                       0               0.00000  \n",
       "3                        859                       0               0.00000  \n",
       "4                         99                       0               0.00000  \n",
       "...                      ...                     ...                   ...  \n",
       "191507               4037882                    4539               0.00112  \n",
       "191508                478701                     166               0.00035  \n",
       "191509               6384919                    5326               0.00083  \n",
       "191510                753611                     189               0.00025  \n",
       "191511               5748545                    5138               0.00089  \n",
       "\n",
       "[191512 rows x 12 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the new column order\n",
    "new_order = [\n",
    "    'academic_year',\n",
    "    'aggregate_level',\n",
    "    'metric_id',\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'charter_yn',\n",
    "    'reporting_category',\n",
    "    'grade',\n",
    "    'cumulative_enrollment',\n",
    "    'expelled_students_total',\n",
    "    'expulsion_rate_total'\n",
    "]\n",
    "expulsion_df = expulsion_df[new_order]\n",
    "expulsion_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>total_all_cumulative_enrollment</th>\n",
       "      <th>total_all_expelled_students_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>4661</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>492</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>4187</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>61119</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>10690</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>01</td>\n",
       "      <td>61119</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>9155</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191456</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>S</td>\n",
       "      <td>07</td>\n",
       "      <td>61796</td>\n",
       "      <td>6057228</td>\n",
       "      <td>All</td>\n",
       "      <td>1046</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191457</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>S</td>\n",
       "      <td>07</td>\n",
       "      <td>61796</td>\n",
       "      <td>6004733</td>\n",
       "      <td>No</td>\n",
       "      <td>385</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191509</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191510</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191511</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>5748545</td>\n",
       "      <td>5138</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14779 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "16           2022-23               D          01         10017     0000000   \n",
       "29           2022-23               D          01         10017     0000000   \n",
       "46           2022-23               D          01         10017     0000000   \n",
       "64           2022-23               D          01         61119     0000000   \n",
       "82           2022-23               D          01         61119     0000000   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "191456       2017-18               S          07         61796     6057228   \n",
       "191457       2017-18               S          07         61796     6004733   \n",
       "191509       2017-18               T          00         00000     0000000   \n",
       "191510       2017-18               T          00         00000     0000000   \n",
       "191511       2017-18               T          00         00000     0000000   \n",
       "\n",
       "       charter_yn total_all_cumulative_enrollment  \\\n",
       "16            All                            4661   \n",
       "29            No                              492   \n",
       "46            Yes                            4187   \n",
       "64            All                           10690   \n",
       "82            No                             9155   \n",
       "...           ...                             ...   \n",
       "191456        All                            1046   \n",
       "191457        No                              385   \n",
       "191509        All                         6384919   \n",
       "191510        Yes                          753611   \n",
       "191511        No                          5748545   \n",
       "\n",
       "       total_all_expelled_students_total  \n",
       "16                                     0  \n",
       "29                                     0  \n",
       "46                                     0  \n",
       "64                                     1  \n",
       "82                                     1  \n",
       "...                                  ...  \n",
       "191456                                 2  \n",
       "191457                                 0  \n",
       "191509                              5326  \n",
       "191510                               189  \n",
       "191511                              5138  \n",
       "\n",
       "[14779 rows x 8 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_figures_exp_df = expulsion_df[expulsion_df['reporting_category'] == 'TA'][['academic_year', 'aggregate_level', 'county_code', 'district_code', 'school_code', 'charter_yn', 'cumulative_enrollment', 'expelled_students_total']]\n",
    "\n",
    "total_figures_exp_df = total_figures_exp_df.rename(columns={\n",
    "    'cumulative_enrollment': 'total_all_cumulative_enrollment',\n",
    "    'expelled_students_total': 'total_all_expelled_students_total'\n",
    "})\n",
    "\n",
    "\n",
    "total_figures_exp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>expelled_students_total</th>\n",
       "      <th>expulsion_rate_total</th>\n",
       "      <th>total_all_cumulative_enrollment</th>\n",
       "      <th>total_all_expelled_students_total</th>\n",
       "      <th>cumulative_enrollment_diff</th>\n",
       "      <th>expelled_students_total_diff</th>\n",
       "      <th>expulsion_rate_total_non_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2233</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>4661</td>\n",
       "      <td>0</td>\n",
       "      <td>2428</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2419</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>4661</td>\n",
       "      <td>0</td>\n",
       "      <td>2242</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>4661</td>\n",
       "      <td>0</td>\n",
       "      <td>3971</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>859</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>4661</td>\n",
       "      <td>0</td>\n",
       "      <td>3802</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>99</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>4661</td>\n",
       "      <td>0</td>\n",
       "      <td>4562</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191359</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>4037882</td>\n",
       "      <td>4539</td>\n",
       "      <td>0.00112</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5326</td>\n",
       "      <td>2347037</td>\n",
       "      <td>787</td>\n",
       "      <td>0.00034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191360</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>478701</td>\n",
       "      <td>166</td>\n",
       "      <td>0.00035</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "      <td>274910</td>\n",
       "      <td>23</td>\n",
       "      <td>0.00008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191361</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5326</td>\n",
       "      <td>0.00083</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5326</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191362</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "      <td>0.00025</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191363</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5748545</td>\n",
       "      <td>5138</td>\n",
       "      <td>0.00089</td>\n",
       "      <td>5748545</td>\n",
       "      <td>5138</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191364 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               D        602          01         10017   \n",
       "1            2022-23               D        602          01         10017   \n",
       "2            2022-23               D        602          01         10017   \n",
       "3            2022-23               D        602          01         10017   \n",
       "4            2022-23               D        602          01         10017   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "191359       2017-18               T        602          00         00000   \n",
       "191360       2017-18               T        602          00         00000   \n",
       "191361       2017-18               T        602          00         00000   \n",
       "191362       2017-18               T        602          00         00000   \n",
       "191363       2017-18               T        602          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  \\\n",
       "0          0000000        All                 GF  All Grades   \n",
       "1          0000000        All                 GM  All Grades   \n",
       "2          0000000        All                 RA  All Grades   \n",
       "3          0000000        All                 RB  All Grades   \n",
       "4          0000000        All                 RD  All Grades   \n",
       "...            ...        ...                ...         ...   \n",
       "191359     0000000        All                 SS  All Grades   \n",
       "191360     0000000        Yes                 SS  All Grades   \n",
       "191361     0000000        All                 TA  All Grades   \n",
       "191362     0000000        Yes                 TA  All Grades   \n",
       "191363     0000000        No                  TA  All Grades   \n",
       "\n",
       "       cumulative_enrollment expelled_students_total  expulsion_rate_total  \\\n",
       "0                       2233                       0               0.00000   \n",
       "1                       2419                       0               0.00000   \n",
       "2                        690                       0               0.00000   \n",
       "3                        859                       0               0.00000   \n",
       "4                         99                       0               0.00000   \n",
       "...                      ...                     ...                   ...   \n",
       "191359               4037882                    4539               0.00112   \n",
       "191360                478701                     166               0.00035   \n",
       "191361               6384919                    5326               0.00083   \n",
       "191362                753611                     189               0.00025   \n",
       "191363               5748545                    5138               0.00089   \n",
       "\n",
       "       total_all_cumulative_enrollment total_all_expelled_students_total  \\\n",
       "0                                 4661                                 0   \n",
       "1                                 4661                                 0   \n",
       "2                                 4661                                 0   \n",
       "3                                 4661                                 0   \n",
       "4                                 4661                                 0   \n",
       "...                                ...                               ...   \n",
       "191359                         6384919                              5326   \n",
       "191360                          753611                               189   \n",
       "191361                         6384919                              5326   \n",
       "191362                          753611                               189   \n",
       "191363                         5748545                              5138   \n",
       "\n",
       "        cumulative_enrollment_diff  expelled_students_total_diff  \\\n",
       "0                             2428                             0   \n",
       "1                             2242                             0   \n",
       "2                             3971                             0   \n",
       "3                             3802                             0   \n",
       "4                             4562                             0   \n",
       "...                            ...                           ...   \n",
       "191359                     2347037                           787   \n",
       "191360                      274910                            23   \n",
       "191361                           0                             0   \n",
       "191362                           0                             0   \n",
       "191363                           0                             0   \n",
       "\n",
       "        expulsion_rate_total_non_group  \n",
       "0                              0.00000  \n",
       "1                              0.00000  \n",
       "2                              0.00000  \n",
       "3                              0.00000  \n",
       "4                              0.00000  \n",
       "...                                ...  \n",
       "191359                         0.00034  \n",
       "191360                         0.00008  \n",
       "191361                             NaN  \n",
       "191362                             NaN  \n",
       "191363                             NaN  \n",
       "\n",
       "[191364 rows x 17 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expulsion_df = expulsion_df.merge(total_figures_exp_df, on=['academic_year', 'aggregate_level', 'county_code', 'district_code', 'school_code', 'charter_yn' ])\n",
    "expulsion_df['cumulative_enrollment_diff'] = expulsion_df['total_all_cumulative_enrollment'].astype(int) - expulsion_df['cumulative_enrollment'].astype(int)\n",
    "expulsion_df['expelled_students_total_diff'] = expulsion_df['total_all_expelled_students_total'].astype(int) - expulsion_df['expelled_students_total'].astype(int)\n",
    "\n",
    "expulsion_df['expulsion_rate_total_non_group'] = (expulsion_df['expelled_students_total_diff'].astype(int) / expulsion_df['cumulative_enrollment_diff'].astype(int) ).round(5)\n",
    "expulsion_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>cumulative_enrollment</th>\n",
       "      <th>expelled_students_total</th>\n",
       "      <th>expulsion_rate_total</th>\n",
       "      <th>expulsion_rate_total_non_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2233</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2419</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>859</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>99</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191359</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>4037882</td>\n",
       "      <td>4539</td>\n",
       "      <td>0.00112</td>\n",
       "      <td>0.00034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191360</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>478701</td>\n",
       "      <td>166</td>\n",
       "      <td>0.00035</td>\n",
       "      <td>0.00008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191361</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5326</td>\n",
       "      <td>0.00083</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191362</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "      <td>0.00025</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191363</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5748545</td>\n",
       "      <td>5138</td>\n",
       "      <td>0.00089</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191364 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               D        602          01         10017   \n",
       "1            2022-23               D        602          01         10017   \n",
       "2            2022-23               D        602          01         10017   \n",
       "3            2022-23               D        602          01         10017   \n",
       "4            2022-23               D        602          01         10017   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "191359       2017-18               T        602          00         00000   \n",
       "191360       2017-18               T        602          00         00000   \n",
       "191361       2017-18               T        602          00         00000   \n",
       "191362       2017-18               T        602          00         00000   \n",
       "191363       2017-18               T        602          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  \\\n",
       "0          0000000        All                 GF  All Grades   \n",
       "1          0000000        All                 GM  All Grades   \n",
       "2          0000000        All                 RA  All Grades   \n",
       "3          0000000        All                 RB  All Grades   \n",
       "4          0000000        All                 RD  All Grades   \n",
       "...            ...        ...                ...         ...   \n",
       "191359     0000000        All                 SS  All Grades   \n",
       "191360     0000000        Yes                 SS  All Grades   \n",
       "191361     0000000        All                 TA  All Grades   \n",
       "191362     0000000        Yes                 TA  All Grades   \n",
       "191363     0000000        No                  TA  All Grades   \n",
       "\n",
       "       cumulative_enrollment expelled_students_total  expulsion_rate_total  \\\n",
       "0                       2233                       0               0.00000   \n",
       "1                       2419                       0               0.00000   \n",
       "2                        690                       0               0.00000   \n",
       "3                        859                       0               0.00000   \n",
       "4                         99                       0               0.00000   \n",
       "...                      ...                     ...                   ...   \n",
       "191359               4037882                    4539               0.00112   \n",
       "191360                478701                     166               0.00035   \n",
       "191361               6384919                    5326               0.00083   \n",
       "191362                753611                     189               0.00025   \n",
       "191363               5748545                    5138               0.00089   \n",
       "\n",
       "        expulsion_rate_total_non_group  \n",
       "0                              0.00000  \n",
       "1                              0.00000  \n",
       "2                              0.00000  \n",
       "3                              0.00000  \n",
       "4                              0.00000  \n",
       "...                                ...  \n",
       "191359                         0.00034  \n",
       "191360                         0.00008  \n",
       "191361                             NaN  \n",
       "191362                             NaN  \n",
       "191363                             NaN  \n",
       "\n",
       "[191364 rows x 13 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the new column order\n",
    "new_order = [\n",
    "    'academic_year',\n",
    "    'aggregate_level',\n",
    "    'metric_id',\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'charter_yn',\n",
    "    'reporting_category',\n",
    "    'grade',\n",
    "    'cumulative_enrollment',\n",
    "    'expelled_students_total',\n",
    "    'expulsion_rate_total',\n",
    "    'expulsion_rate_total_non_group'\n",
    "]\n",
    "expulsion_df = expulsion_df[new_order]\n",
    "expulsion_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chronic Absenteeism "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>county_name</th>\n",
       "      <th>district_name</th>\n",
       "      <th>school_name</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>dass</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>enrollment_count</th>\n",
       "      <th>absenteeism_count</th>\n",
       "      <th>absenteeism_rate</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>2893518</td>\n",
       "      <td>719562</td>\n",
       "      <td>24.9</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>3059625</td>\n",
       "      <td>764487</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>GR13</td>\n",
       "      <td>1266699</td>\n",
       "      <td>309500</td>\n",
       "      <td>24.4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>GR46</td>\n",
       "      <td>1318062</td>\n",
       "      <td>270008</td>\n",
       "      <td>20.5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>GR78</td>\n",
       "      <td>888084</td>\n",
       "      <td>202692</td>\n",
       "      <td>22.8</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208770</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NV</td>\n",
       "      <td>SS</td>\n",
       "      <td>3610723</td>\n",
       "      <td>499336</td>\n",
       "      <td>13.8</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208771</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NV</td>\n",
       "      <td>SS</td>\n",
       "      <td>438944</td>\n",
       "      <td>73929</td>\n",
       "      <td>16.8</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208772</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>NV</td>\n",
       "      <td>TA</td>\n",
       "      <td>6315131</td>\n",
       "      <td>702531</td>\n",
       "      <td>11.1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208773</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NV</td>\n",
       "      <td>TA</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "      <td>11.1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208774</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NV</td>\n",
       "      <td>TA</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "      <td>13.4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>208775 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "0            2022-23               T          00           NaN         NaN   \n",
       "1            2022-23               T          00           NaN         NaN   \n",
       "2            2022-23               T          00           NaN         NaN   \n",
       "3            2022-23               T          00           NaN         NaN   \n",
       "4            2022-23               T          00           NaN         NaN   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "208770       2017-18               T          00           NaN         NaN   \n",
       "208771       2017-18               T          00           NaN         NaN   \n",
       "208772       2017-18               T          00           NaN         NaN   \n",
       "208773       2017-18               T          00           NaN         NaN   \n",
       "208774       2017-18               T          00           NaN         NaN   \n",
       "\n",
       "       county_name district_name school_name charter_yn dass  \\\n",
       "0            State           NaN         NaN        All  All   \n",
       "1            State           NaN         NaN        All  All   \n",
       "2            State           NaN         NaN        All  All   \n",
       "3            State           NaN         NaN        All  All   \n",
       "4            State           NaN         NaN        All  All   \n",
       "...            ...           ...         ...        ...  ...   \n",
       "208770       State           NaN         NaN         No   NV   \n",
       "208771       State           NaN         NaN        Yes   NV   \n",
       "208772       State           NaN         NaN        All   NV   \n",
       "208773       State           NaN         NaN         No   NV   \n",
       "208774       State           NaN         NaN        Yes   NV   \n",
       "\n",
       "       reporting_category enrollment_count absenteeism_count absenteeism_rate  \\\n",
       "0                      GF          2893518            719562             24.9   \n",
       "1                      GM          3059625            764487               25   \n",
       "2                    GR13          1266699            309500             24.4   \n",
       "3                    GR46          1318062            270008             20.5   \n",
       "4                    GR78           888084            202692             22.8   \n",
       "...                   ...              ...               ...              ...   \n",
       "208770                 SS          3610723            499336             13.8   \n",
       "208771                 SS           438944             73929             16.8   \n",
       "208772                 TA          6315131            702531             11.1   \n",
       "208773                 TA          5688600            629719             11.1   \n",
       "208774                 TA           690446             92822             13.4   \n",
       "\n",
       "       is_wcc_entity  \n",
       "0              False  \n",
       "1              False  \n",
       "2              False  \n",
       "3              False  \n",
       "4              False  \n",
       "...              ...  \n",
       "208770         False  \n",
       "208771         False  \n",
       "208772         False  \n",
       "208773         False  \n",
       "208774         False  \n",
       "\n",
       "[208775 rows x 15 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "absenteeism_df = pd.read_csv('final_long_datasets_domain/final_chronic_absenteeism_df.csv', encoding='ISO-8859-1',dtype=str)\n",
    "\n",
    "absenteeism_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "absenteeism_df['charter_yn'] = absenteeism_df['charter_yn'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['All', 'No', 'Yes'], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "absenteeism_df.charter_yn.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of values to filter out\n",
    "values_to_remove = ['GR13', 'GR46', 'GR78', 'GR912', 'GRK8', 'GRKN', 'GRK', 'GRUG']\n",
    "\n",
    "# Filter out rows where reporting_category column has any of the values in values_to_remove\n",
    "absenteeism_df = absenteeism_df[~absenteeism_df['reporting_category'].isin(values_to_remove)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['GF', 'GM', 'GX', 'RA', 'RB', 'RD', 'RF', 'RH', 'RI', 'RP', 'RT',\n",
       "       'RW', 'SD', 'SE', 'SF', 'SH', 'SM', 'SS', 'TA'], dtype=object)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "absenteeism_df.reporting_category.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/6t/n3rqwc2x06ngf2r8521g1nmc0000gn/T/ipykernel_20873/1384718005.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  absenteeism_df['metric_id'] = 502\n",
      "/var/folders/6t/n3rqwc2x06ngf2r8521g1nmc0000gn/T/ipykernel_20873/1384718005.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  absenteeism_df['grade'] = 'All Grades'\n",
      "/var/folders/6t/n3rqwc2x06ngf2r8521g1nmc0000gn/T/ipykernel_20873/1384718005.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  absenteeism_df['absenteeism_rate'] = (absenteeism_df['absenteeism_count'].astype(int) / absenteeism_df['enrollment_count'].astype(int)).round(5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>county_name</th>\n",
       "      <th>district_name</th>\n",
       "      <th>school_name</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>dass</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>enrollment_count</th>\n",
       "      <th>absenteeism_count</th>\n",
       "      <th>absenteeism_rate</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>grade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>2893518</td>\n",
       "      <td>719562</td>\n",
       "      <td>0.24868</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>3059625</td>\n",
       "      <td>764487</td>\n",
       "      <td>0.24986</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>GX</td>\n",
       "      <td>5746</td>\n",
       "      <td>2253</td>\n",
       "      <td>0.39210</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>578202</td>\n",
       "      <td>58964</td>\n",
       "      <td>0.10198</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>301921</td>\n",
       "      <td>110537</td>\n",
       "      <td>0.36611</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208770</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NV</td>\n",
       "      <td>SS</td>\n",
       "      <td>3610723</td>\n",
       "      <td>499336</td>\n",
       "      <td>0.13829</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208771</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NV</td>\n",
       "      <td>SS</td>\n",
       "      <td>438944</td>\n",
       "      <td>73929</td>\n",
       "      <td>0.16842</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208772</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>NV</td>\n",
       "      <td>TA</td>\n",
       "      <td>6315131</td>\n",
       "      <td>702531</td>\n",
       "      <td>0.11125</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208773</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NV</td>\n",
       "      <td>TA</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "      <td>0.11070</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208774</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NV</td>\n",
       "      <td>TA</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "      <td>0.13444</td>\n",
       "      <td>False</td>\n",
       "      <td>502</td>\n",
       "      <td>All Grades</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150455 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "0            2022-23               T          00         00000     0000000   \n",
       "1            2022-23               T          00         00000     0000000   \n",
       "8            2022-23               T          00         00000     0000000   \n",
       "9            2022-23               T          00         00000     0000000   \n",
       "10           2022-23               T          00         00000     0000000   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "208770       2017-18               T          00         00000     0000000   \n",
       "208771       2017-18               T          00         00000     0000000   \n",
       "208772       2017-18               T          00         00000     0000000   \n",
       "208773       2017-18               T          00         00000     0000000   \n",
       "208774       2017-18               T          00         00000     0000000   \n",
       "\n",
       "       county_name district_name school_name charter_yn dass  \\\n",
       "0            State           NaN         NaN        All  All   \n",
       "1            State           NaN         NaN        All  All   \n",
       "8            State           NaN         NaN        All  All   \n",
       "9            State           NaN         NaN        All  All   \n",
       "10           State           NaN         NaN        All  All   \n",
       "...            ...           ...         ...        ...  ...   \n",
       "208770       State           NaN         NaN         No   NV   \n",
       "208771       State           NaN         NaN        Yes   NV   \n",
       "208772       State           NaN         NaN        All   NV   \n",
       "208773       State           NaN         NaN         No   NV   \n",
       "208774       State           NaN         NaN        Yes   NV   \n",
       "\n",
       "       reporting_category enrollment_count absenteeism_count  \\\n",
       "0                      GF          2893518            719562   \n",
       "1                      GM          3059625            764487   \n",
       "8                      GX             5746              2253   \n",
       "9                      RA           578202             58964   \n",
       "10                     RB           301921            110537   \n",
       "...                   ...              ...               ...   \n",
       "208770                 SS          3610723            499336   \n",
       "208771                 SS           438944             73929   \n",
       "208772                 TA          6315131            702531   \n",
       "208773                 TA          5688600            629719   \n",
       "208774                 TA           690446             92822   \n",
       "\n",
       "        absenteeism_rate is_wcc_entity  metric_id       grade  \n",
       "0                0.24868         False        502  All Grades  \n",
       "1                0.24986         False        502  All Grades  \n",
       "8                0.39210         False        502  All Grades  \n",
       "9                0.10198         False        502  All Grades  \n",
       "10               0.36611         False        502  All Grades  \n",
       "...                  ...           ...        ...         ...  \n",
       "208770           0.13829         False        502  All Grades  \n",
       "208771           0.16842         False        502  All Grades  \n",
       "208772           0.11125         False        502  All Grades  \n",
       "208773           0.11070         False        502  All Grades  \n",
       "208774           0.13444         False        502  All Grades  \n",
       "\n",
       "[150455 rows x 17 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "absenteeism_df.loc[absenteeism_df['aggregate_level'] == 'T', 'district_code'] = '00000'\n",
    "absenteeism_df.loc[absenteeism_df['aggregate_level'] == 'T', 'school_code'] = '0000000'\n",
    "absenteeism_df['metric_id'] = 502\n",
    "absenteeism_df['grade'] = 'All Grades'\n",
    "\n",
    "\n",
    "absenteeism_df['absenteeism_rate'] = (absenteeism_df['absenteeism_count'].astype(int) / absenteeism_df['enrollment_count'].astype(int)).round(5)\n",
    "absenteeism_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>enrollment_count</th>\n",
       "      <th>absenteeism_count</th>\n",
       "      <th>absenteeism_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2893518</td>\n",
       "      <td>719562</td>\n",
       "      <td>0.24868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3059625</td>\n",
       "      <td>764487</td>\n",
       "      <td>0.24986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GX</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5746</td>\n",
       "      <td>2253</td>\n",
       "      <td>0.39210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>578202</td>\n",
       "      <td>58964</td>\n",
       "      <td>0.10198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>301921</td>\n",
       "      <td>110537</td>\n",
       "      <td>0.36611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208770</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3610723</td>\n",
       "      <td>499336</td>\n",
       "      <td>0.13829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208771</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>438944</td>\n",
       "      <td>73929</td>\n",
       "      <td>0.16842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208772</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6315131</td>\n",
       "      <td>702531</td>\n",
       "      <td>0.11125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208773</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "      <td>0.11070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208774</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "      <td>0.13444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150455 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               T        502          00         00000   \n",
       "1            2022-23               T        502          00         00000   \n",
       "8            2022-23               T        502          00         00000   \n",
       "9            2022-23               T        502          00         00000   \n",
       "10           2022-23               T        502          00         00000   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "208770       2017-18               T        502          00         00000   \n",
       "208771       2017-18               T        502          00         00000   \n",
       "208772       2017-18               T        502          00         00000   \n",
       "208773       2017-18               T        502          00         00000   \n",
       "208774       2017-18               T        502          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade enrollment_count  \\\n",
       "0          0000000        All                 GF  All Grades          2893518   \n",
       "1          0000000        All                 GM  All Grades          3059625   \n",
       "8          0000000        All                 GX  All Grades             5746   \n",
       "9          0000000        All                 RA  All Grades           578202   \n",
       "10         0000000        All                 RB  All Grades           301921   \n",
       "...            ...        ...                ...         ...              ...   \n",
       "208770     0000000         No                 SS  All Grades          3610723   \n",
       "208771     0000000        Yes                 SS  All Grades           438944   \n",
       "208772     0000000        All                 TA  All Grades          6315131   \n",
       "208773     0000000         No                 TA  All Grades          5688600   \n",
       "208774     0000000        Yes                 TA  All Grades           690446   \n",
       "\n",
       "       absenteeism_count  absenteeism_rate  \n",
       "0                 719562           0.24868  \n",
       "1                 764487           0.24986  \n",
       "8                   2253           0.39210  \n",
       "9                  58964           0.10198  \n",
       "10                110537           0.36611  \n",
       "...                  ...               ...  \n",
       "208770            499336           0.13829  \n",
       "208771             73929           0.16842  \n",
       "208772            702531           0.11125  \n",
       "208773            629719           0.11070  \n",
       "208774             92822           0.13444  \n",
       "\n",
       "[150455 rows x 12 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the new column order\n",
    "new_order = [\n",
    "    'academic_year',\n",
    "    'aggregate_level',\n",
    "    'metric_id',\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'charter_yn',\n",
    "    'reporting_category',\n",
    "    'grade',\n",
    "    'enrollment_count',\n",
    "    'absenteeism_count',\n",
    "    'absenteeism_rate'\n",
    "]\n",
    "absenteeism_df = absenteeism_df[new_order]\n",
    "absenteeism_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>total_all_enrollment_count</th>\n",
       "      <th>total_all_absenteeism_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>5958889</td>\n",
       "      <td>1486302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>5268302</td>\n",
       "      <td>1339670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>743502</td>\n",
       "      <td>169748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>S</td>\n",
       "      <td>07</td>\n",
       "      <td>10074</td>\n",
       "      <td>0114470</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1135</td>\n",
       "      <td>262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>S</td>\n",
       "      <td>07</td>\n",
       "      <td>10074</td>\n",
       "      <td>0129528</td>\n",
       "      <td>Yes</td>\n",
       "      <td>924</td>\n",
       "      <td>329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208698</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>S</td>\n",
       "      <td>07</td>\n",
       "      <td>61796</td>\n",
       "      <td>6119515</td>\n",
       "      <td>All</td>\n",
       "      <td>619</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208699</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>S</td>\n",
       "      <td>07</td>\n",
       "      <td>61796</td>\n",
       "      <td>6120885</td>\n",
       "      <td>All</td>\n",
       "      <td>485</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208772</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>6315131</td>\n",
       "      <td>702531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208773</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208774</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12113 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level county_code district_code school_code  \\\n",
       "24           2022-23               T          00         00000     0000000   \n",
       "49           2022-23               T          00         00000     0000000   \n",
       "74           2022-23               T          00         00000     0000000   \n",
       "88           2022-23               S          07         10074     0114470   \n",
       "105          2022-23               S          07         10074     0129528   \n",
       "...              ...             ...         ...           ...         ...   \n",
       "208698       2017-18               S          07         61796     6119515   \n",
       "208699       2017-18               S          07         61796     6120885   \n",
       "208772       2017-18               T          00         00000     0000000   \n",
       "208773       2017-18               T          00         00000     0000000   \n",
       "208774       2017-18               T          00         00000     0000000   \n",
       "\n",
       "       charter_yn total_all_enrollment_count total_all_absenteeism_count  \n",
       "24            All                    5958889                     1486302  \n",
       "49             No                    5268302                     1339670  \n",
       "74            Yes                     743502                      169748  \n",
       "88            Yes                       1135                         262  \n",
       "105           Yes                        924                         329  \n",
       "...           ...                        ...                         ...  \n",
       "208698        All                        619                          26  \n",
       "208699        All                        485                         104  \n",
       "208772        All                    6315131                      702531  \n",
       "208773         No                    5688600                      629719  \n",
       "208774        Yes                     690446                       92822  \n",
       "\n",
       "[12113 rows x 8 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_figures_abs_df = absenteeism_df[absenteeism_df['reporting_category'] == 'TA'][['academic_year', 'aggregate_level', 'county_code', 'district_code', 'school_code', 'charter_yn', 'enrollment_count', 'absenteeism_count']]\n",
    "\n",
    "total_figures_abs_df = total_figures_abs_df.rename(columns={\n",
    "    'enrollment_count': 'total_all_enrollment_count',\n",
    "    'absenteeism_count': 'total_all_absenteeism_count'\n",
    "})\n",
    "\n",
    "total_figures_abs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>enrollment_count</th>\n",
       "      <th>absenteeism_count</th>\n",
       "      <th>absenteeism_rate</th>\n",
       "      <th>total_all_enrollment_count</th>\n",
       "      <th>total_all_absenteeism_count</th>\n",
       "      <th>enrollment_count_diff</th>\n",
       "      <th>absenteeism_count_diff</th>\n",
       "      <th>absenteeism_rate_non_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2893518</td>\n",
       "      <td>719562</td>\n",
       "      <td>0.24868</td>\n",
       "      <td>5958889</td>\n",
       "      <td>1486302</td>\n",
       "      <td>3065371</td>\n",
       "      <td>766740</td>\n",
       "      <td>0.25013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3059625</td>\n",
       "      <td>764487</td>\n",
       "      <td>0.24986</td>\n",
       "      <td>5958889</td>\n",
       "      <td>1486302</td>\n",
       "      <td>2899264</td>\n",
       "      <td>721815</td>\n",
       "      <td>0.24896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GX</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5746</td>\n",
       "      <td>2253</td>\n",
       "      <td>0.39210</td>\n",
       "      <td>5958889</td>\n",
       "      <td>1486302</td>\n",
       "      <td>5953143</td>\n",
       "      <td>1484049</td>\n",
       "      <td>0.24929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>578202</td>\n",
       "      <td>58964</td>\n",
       "      <td>0.10198</td>\n",
       "      <td>5958889</td>\n",
       "      <td>1486302</td>\n",
       "      <td>5380687</td>\n",
       "      <td>1427338</td>\n",
       "      <td>0.26527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>301921</td>\n",
       "      <td>110537</td>\n",
       "      <td>0.36611</td>\n",
       "      <td>5958889</td>\n",
       "      <td>1486302</td>\n",
       "      <td>5656968</td>\n",
       "      <td>1375765</td>\n",
       "      <td>0.24320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150450</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3610723</td>\n",
       "      <td>499336</td>\n",
       "      <td>0.13829</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "      <td>2077877</td>\n",
       "      <td>130383</td>\n",
       "      <td>0.06275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150451</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>438944</td>\n",
       "      <td>73929</td>\n",
       "      <td>0.16842</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "      <td>251502</td>\n",
       "      <td>18893</td>\n",
       "      <td>0.07512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150452</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6315131</td>\n",
       "      <td>702531</td>\n",
       "      <td>0.11125</td>\n",
       "      <td>6315131</td>\n",
       "      <td>702531</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150453</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "      <td>0.11070</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150454</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "      <td>0.13444</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150455 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               T        502          00         00000   \n",
       "1            2022-23               T        502          00         00000   \n",
       "2            2022-23               T        502          00         00000   \n",
       "3            2022-23               T        502          00         00000   \n",
       "4            2022-23               T        502          00         00000   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "150450       2017-18               T        502          00         00000   \n",
       "150451       2017-18               T        502          00         00000   \n",
       "150452       2017-18               T        502          00         00000   \n",
       "150453       2017-18               T        502          00         00000   \n",
       "150454       2017-18               T        502          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade enrollment_count  \\\n",
       "0          0000000        All                 GF  All Grades          2893518   \n",
       "1          0000000        All                 GM  All Grades          3059625   \n",
       "2          0000000        All                 GX  All Grades             5746   \n",
       "3          0000000        All                 RA  All Grades           578202   \n",
       "4          0000000        All                 RB  All Grades           301921   \n",
       "...            ...        ...                ...         ...              ...   \n",
       "150450     0000000         No                 SS  All Grades          3610723   \n",
       "150451     0000000        Yes                 SS  All Grades           438944   \n",
       "150452     0000000        All                 TA  All Grades          6315131   \n",
       "150453     0000000         No                 TA  All Grades          5688600   \n",
       "150454     0000000        Yes                 TA  All Grades           690446   \n",
       "\n",
       "       absenteeism_count  absenteeism_rate total_all_enrollment_count  \\\n",
       "0                 719562           0.24868                    5958889   \n",
       "1                 764487           0.24986                    5958889   \n",
       "2                   2253           0.39210                    5958889   \n",
       "3                  58964           0.10198                    5958889   \n",
       "4                 110537           0.36611                    5958889   \n",
       "...                  ...               ...                        ...   \n",
       "150450            499336           0.13829                    5688600   \n",
       "150451             73929           0.16842                     690446   \n",
       "150452            702531           0.11125                    6315131   \n",
       "150453            629719           0.11070                    5688600   \n",
       "150454             92822           0.13444                     690446   \n",
       "\n",
       "       total_all_absenteeism_count  enrollment_count_diff  \\\n",
       "0                          1486302                3065371   \n",
       "1                          1486302                2899264   \n",
       "2                          1486302                5953143   \n",
       "3                          1486302                5380687   \n",
       "4                          1486302                5656968   \n",
       "...                            ...                    ...   \n",
       "150450                      629719                2077877   \n",
       "150451                       92822                 251502   \n",
       "150452                      702531                      0   \n",
       "150453                      629719                      0   \n",
       "150454                       92822                      0   \n",
       "\n",
       "        absenteeism_count_diff  absenteeism_rate_non_group  \n",
       "0                       766740                     0.25013  \n",
       "1                       721815                     0.24896  \n",
       "2                      1484049                     0.24929  \n",
       "3                      1427338                     0.26527  \n",
       "4                      1375765                     0.24320  \n",
       "...                        ...                         ...  \n",
       "150450                  130383                     0.06275  \n",
       "150451                   18893                     0.07512  \n",
       "150452                       0                         NaN  \n",
       "150453                       0                         NaN  \n",
       "150454                       0                         NaN  \n",
       "\n",
       "[150455 rows x 17 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "absenteeism_df = absenteeism_df.merge(total_figures_abs_df, on=['academic_year', 'aggregate_level', 'county_code', 'district_code', 'school_code', 'charter_yn'])\n",
    "absenteeism_df['enrollment_count_diff'] = absenteeism_df['total_all_enrollment_count'].astype(int) - absenteeism_df['enrollment_count'].astype(int)\n",
    "absenteeism_df['absenteeism_count_diff'] = absenteeism_df['total_all_absenteeism_count'].astype(int) - absenteeism_df['absenteeism_count'].astype(int)\n",
    "\n",
    "absenteeism_df['absenteeism_rate_non_group'] = (absenteeism_df['absenteeism_count_diff'].astype(int) / absenteeism_df['enrollment_count_diff'].astype(int) ).round(5)\n",
    "absenteeism_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>enrollment_count</th>\n",
       "      <th>absenteeism_count</th>\n",
       "      <th>absenteeism_rate</th>\n",
       "      <th>absenteeism_rate_non_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2893518</td>\n",
       "      <td>719562</td>\n",
       "      <td>0.24868</td>\n",
       "      <td>0.25013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3059625</td>\n",
       "      <td>764487</td>\n",
       "      <td>0.24986</td>\n",
       "      <td>0.24896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GX</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5746</td>\n",
       "      <td>2253</td>\n",
       "      <td>0.39210</td>\n",
       "      <td>0.24929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>578202</td>\n",
       "      <td>58964</td>\n",
       "      <td>0.10198</td>\n",
       "      <td>0.26527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>301921</td>\n",
       "      <td>110537</td>\n",
       "      <td>0.36611</td>\n",
       "      <td>0.24320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150450</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3610723</td>\n",
       "      <td>499336</td>\n",
       "      <td>0.13829</td>\n",
       "      <td>0.06275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150451</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>438944</td>\n",
       "      <td>73929</td>\n",
       "      <td>0.16842</td>\n",
       "      <td>0.07512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150452</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6315131</td>\n",
       "      <td>702531</td>\n",
       "      <td>0.11125</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150453</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "      <td>0.11070</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150454</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "      <td>0.13444</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150455 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               T        502          00         00000   \n",
       "1            2022-23               T        502          00         00000   \n",
       "2            2022-23               T        502          00         00000   \n",
       "3            2022-23               T        502          00         00000   \n",
       "4            2022-23               T        502          00         00000   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "150450       2017-18               T        502          00         00000   \n",
       "150451       2017-18               T        502          00         00000   \n",
       "150452       2017-18               T        502          00         00000   \n",
       "150453       2017-18               T        502          00         00000   \n",
       "150454       2017-18               T        502          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade enrollment_count  \\\n",
       "0          0000000        All                 GF  All Grades          2893518   \n",
       "1          0000000        All                 GM  All Grades          3059625   \n",
       "2          0000000        All                 GX  All Grades             5746   \n",
       "3          0000000        All                 RA  All Grades           578202   \n",
       "4          0000000        All                 RB  All Grades           301921   \n",
       "...            ...        ...                ...         ...              ...   \n",
       "150450     0000000         No                 SS  All Grades          3610723   \n",
       "150451     0000000        Yes                 SS  All Grades           438944   \n",
       "150452     0000000        All                 TA  All Grades          6315131   \n",
       "150453     0000000         No                 TA  All Grades          5688600   \n",
       "150454     0000000        Yes                 TA  All Grades           690446   \n",
       "\n",
       "       absenteeism_count  absenteeism_rate  absenteeism_rate_non_group  \n",
       "0                 719562           0.24868                     0.25013  \n",
       "1                 764487           0.24986                     0.24896  \n",
       "2                   2253           0.39210                     0.24929  \n",
       "3                  58964           0.10198                     0.26527  \n",
       "4                 110537           0.36611                     0.24320  \n",
       "...                  ...               ...                         ...  \n",
       "150450            499336           0.13829                     0.06275  \n",
       "150451             73929           0.16842                     0.07512  \n",
       "150452            702531           0.11125                         NaN  \n",
       "150453            629719           0.11070                         NaN  \n",
       "150454             92822           0.13444                         NaN  \n",
       "\n",
       "[150455 rows x 13 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the new column order\n",
    "new_order = [\n",
    "    'academic_year',\n",
    "    'aggregate_level',\n",
    "    'metric_id',\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'charter_yn',\n",
    "    'reporting_category',\n",
    "    'grade',\n",
    "    'enrollment_count',\n",
    "    'absenteeism_count',\n",
    "    'absenteeism_rate',\n",
    "    'absenteeism_rate_non_group'\n",
    "]\n",
    "absenteeism_df = absenteeism_df[new_order]\n",
    "absenteeism_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CAASPP Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2960990</td>\n",
       "      <td>2957491</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>420238</td>\n",
       "      <td>410559</td>\n",
       "      <td>2409.8</td>\n",
       "      <td>23.40</td>\n",
       "      <td>19.56</td>\n",
       "      <td>42.96</td>\n",
       "      <td>22.40</td>\n",
       "      <td>34.64</td>\n",
       "      <td>410290</td>\n",
       "      <td>17.47</td>\n",
       "      <td>57.69</td>\n",
       "      <td>24.84</td>\n",
       "      <td>16.18</td>\n",
       "      <td>50.24</td>\n",
       "      <td>33.58</td>\n",
       "      <td>11.75</td>\n",
       "      <td>70.96</td>\n",
       "      <td>17.29</td>\n",
       "      <td>16.07</td>\n",
       "      <td>60.00</td>\n",
       "      <td>23.94</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1512977</td>\n",
       "      <td>1511175</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>214436</td>\n",
       "      <td>209138</td>\n",
       "      <td>2402.4</td>\n",
       "      <td>21.27</td>\n",
       "      <td>18.77</td>\n",
       "      <td>40.04</td>\n",
       "      <td>22.45</td>\n",
       "      <td>37.51</td>\n",
       "      <td>208998</td>\n",
       "      <td>15.88</td>\n",
       "      <td>57.23</td>\n",
       "      <td>26.89</td>\n",
       "      <td>13.96</td>\n",
       "      <td>49.35</td>\n",
       "      <td>36.69</td>\n",
       "      <td>11.47</td>\n",
       "      <td>70.55</td>\n",
       "      <td>17.99</td>\n",
       "      <td>15.20</td>\n",
       "      <td>59.33</td>\n",
       "      <td>25.47</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1445927</td>\n",
       "      <td>1444241</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>205715</td>\n",
       "      <td>201341</td>\n",
       "      <td>2417.5</td>\n",
       "      <td>25.60</td>\n",
       "      <td>20.39</td>\n",
       "      <td>45.99</td>\n",
       "      <td>22.34</td>\n",
       "      <td>31.67</td>\n",
       "      <td>201212</td>\n",
       "      <td>19.12</td>\n",
       "      <td>58.17</td>\n",
       "      <td>22.71</td>\n",
       "      <td>18.48</td>\n",
       "      <td>51.16</td>\n",
       "      <td>30.36</td>\n",
       "      <td>12.04</td>\n",
       "      <td>71.39</td>\n",
       "      <td>16.57</td>\n",
       "      <td>16.96</td>\n",
       "      <td>60.69</td>\n",
       "      <td>22.35</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>8</td>\n",
       "      <td>B</td>\n",
       "      <td>557974</td>\n",
       "      <td>557303</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>24566</td>\n",
       "      <td>24451</td>\n",
       "      <td>2474.8</td>\n",
       "      <td>43.01</td>\n",
       "      <td>29.94</td>\n",
       "      <td>72.96</td>\n",
       "      <td>19.26</td>\n",
       "      <td>7.78</td>\n",
       "      <td>24435</td>\n",
       "      <td>29.83</td>\n",
       "      <td>63.91</td>\n",
       "      <td>6.26</td>\n",
       "      <td>29.77</td>\n",
       "      <td>60.20</td>\n",
       "      <td>10.02</td>\n",
       "      <td>18.18</td>\n",
       "      <td>74.83</td>\n",
       "      <td>6.98</td>\n",
       "      <td>28.87</td>\n",
       "      <td>64.16</td>\n",
       "      <td>6.97</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>28</td>\n",
       "      <td>B</td>\n",
       "      <td>22322</td>\n",
       "      <td>22290</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3265</td>\n",
       "      <td>3165</td>\n",
       "      <td>2344.5</td>\n",
       "      <td>7.30</td>\n",
       "      <td>10.30</td>\n",
       "      <td>17.60</td>\n",
       "      <td>21.04</td>\n",
       "      <td>61.36</td>\n",
       "      <td>3165</td>\n",
       "      <td>5.81</td>\n",
       "      <td>48.34</td>\n",
       "      <td>45.85</td>\n",
       "      <td>4.84</td>\n",
       "      <td>37.61</td>\n",
       "      <td>57.55</td>\n",
       "      <td>4.96</td>\n",
       "      <td>65.91</td>\n",
       "      <td>29.13</td>\n",
       "      <td>6.32</td>\n",
       "      <td>51.53</td>\n",
       "      <td>42.15</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>128</td>\n",
       "      <td>B</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.76</td>\n",
       "      <td>88.24</td>\n",
       "      <td>17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.76</td>\n",
       "      <td>88.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17.65</td>\n",
       "      <td>82.35</td>\n",
       "      <td>0.00</td>\n",
       "      <td>23.53</td>\n",
       "      <td>76.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2586.9</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2550.0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>20</td>\n",
       "      <td>15.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>45.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>20</td>\n",
       "      <td>15.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>45.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       county_code district_code school_code filler test_year  \\\n",
       "0               00         00000     0000000    NaN      2023   \n",
       "1               00         00000     0000000    NaN      2023   \n",
       "2               00         00000     0000000    NaN      2023   \n",
       "3               00         00000     0000000    NaN      2023   \n",
       "4               00         00000     0000000    NaN      2023   \n",
       "...            ...           ...         ...    ...       ...   \n",
       "524547          58         72769     0000000    NaN      2018   \n",
       "524548          58         72769     0000000    NaN      2018   \n",
       "524549          58         72769     0000000    NaN      2018   \n",
       "524550          58         72769     0000000    NaN      2018   \n",
       "524551          58         72769     0000000    NaN      2018   \n",
       "\n",
       "       student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                     1         B                    2960990   \n",
       "1                     3         B                    1512977   \n",
       "2                     4         B                    1445927   \n",
       "3                     8         B                     557974   \n",
       "4                    28         B                      22322   \n",
       "...                 ...       ...                        ...   \n",
       "524547              128         B                         17   \n",
       "524548              144         B                         20   \n",
       "524549              144         B                         20   \n",
       "524550              144         B                         20   \n",
       "524551              144         B                         20   \n",
       "\n",
       "       total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                       2957491     3       1            420238   \n",
       "1                       1511175     3       1            214436   \n",
       "2                       1444241     3       1            205715   \n",
       "3                        557303     3       1             24566   \n",
       "4                         22290     3       1              3265   \n",
       "...                         ...   ...     ...               ...   \n",
       "524547                       17    13       2                18   \n",
       "524548                       20    11       1                20   \n",
       "524549                       20    11       2                20   \n",
       "524550                       20    13       2                20   \n",
       "524551                       20    13       1                20   \n",
       "\n",
       "       students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0               410559           2409.8            23.40       19.56   \n",
       "1               209138           2402.4            21.27       18.77   \n",
       "2               201341           2417.5            25.60       20.39   \n",
       "3                24451           2474.8            43.01       29.94   \n",
       "4                 3165           2344.5             7.30       10.30   \n",
       "...                ...              ...              ...         ...   \n",
       "524547              17              NaN             0.00        0.00   \n",
       "524548              20           2586.9            30.00       20.00   \n",
       "524549              20           2550.0            10.00       20.00   \n",
       "524550              20              NaN            10.00       20.00   \n",
       "524551              20              NaN            30.00       20.00   \n",
       "\n",
       "       pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                  42.96              22.40           34.64   \n",
       "1                  40.04              22.45           37.51   \n",
       "2                  45.99              22.34           31.67   \n",
       "3                  72.96              19.26            7.78   \n",
       "4                  17.60              21.04           61.36   \n",
       "...                  ...                ...             ...   \n",
       "524547              0.00              11.76           88.24   \n",
       "524548             50.00              25.00           25.00   \n",
       "524549             30.00              35.00           35.00   \n",
       "524550             30.00              35.00           35.00   \n",
       "524551             50.00              25.00           25.00   \n",
       "\n",
       "       students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                    410290                17.47               57.69   \n",
       "1                    208998                15.88               57.23   \n",
       "2                    201212                19.12               58.17   \n",
       "3                     24435                29.83               63.91   \n",
       "4                      3165                 5.81               48.34   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                   17                 0.00               11.76   \n",
       "524548                   20                45.00               30.00   \n",
       "524549                   20                15.00               40.00   \n",
       "524550                   20                15.00               40.00   \n",
       "524551                   20                45.00               30.00   \n",
       "\n",
       "       area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                     24.84                16.18               50.24   \n",
       "1                     26.89                13.96               49.35   \n",
       "2                     22.71                18.48               51.16   \n",
       "3                      6.26                29.77               60.20   \n",
       "4                     45.85                 4.84               37.61   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                88.24                 0.00               17.65   \n",
       "524548                25.00                35.00               30.00   \n",
       "524549                45.00                25.00               35.00   \n",
       "524550                45.00                25.00               35.00   \n",
       "524551                25.00                35.00               30.00   \n",
       "\n",
       "       area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                     33.58                11.75               70.96   \n",
       "1                     36.69                11.47               70.55   \n",
       "2                     30.36                12.04               71.39   \n",
       "3                     10.02                18.18               74.83   \n",
       "4                     57.55                 4.96               65.91   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                82.35                 0.00               23.53   \n",
       "524548                35.00                25.00               65.00   \n",
       "524549                40.00                25.00               35.00   \n",
       "524550                40.00                25.00               35.00   \n",
       "524551                35.00                25.00               65.00   \n",
       "\n",
       "       area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                     17.29                16.07               60.00   \n",
       "1                     17.99                15.20               59.33   \n",
       "2                     16.57                16.96               60.69   \n",
       "3                      6.98                28.87               64.16   \n",
       "4                     29.13                 6.32               51.53   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                76.47                 0.00                0.00   \n",
       "524548                10.00                30.00               35.00   \n",
       "524549                40.00                 0.00                0.00   \n",
       "524550                40.00                 0.00                0.00   \n",
       "524551                10.00                30.00               35.00   \n",
       "\n",
       "       area_4_pct_below_std type_id is_wcc_entity  \n",
       "0                     23.94       4         False  \n",
       "1                     25.47       4         False  \n",
       "2                     22.35       4         False  \n",
       "3                      6.97       4         False  \n",
       "4                     42.15       4         False  \n",
       "...                     ...     ...           ...  \n",
       "524547                 0.00       6         False  \n",
       "524548                35.00       6         False  \n",
       "524549                 0.00       6         False  \n",
       "524550                 0.00       6         False  \n",
       "524551                35.00       6         False  \n",
       "\n",
       "[524552 rows x 34 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df = pd.read_csv('final_long_datasets_domain/final_caaspp_all_grades_df.csv', encoding='ISO-8859-1',dtype=str)\n",
    "\n",
    "caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to determine the aggregate level\n",
    "def determine_aggregate_level(row):\n",
    "    if row['county_code'] == '00':\n",
    "        return 'T'\n",
    "    elif row['district_code'] != '00000' and row['school_code'] == '0000000':\n",
    "        return 'D'\n",
    "    elif row['district_code'] != '00000' and row['school_code'] != '0000000':\n",
    "        return 'S'\n",
    "    else:\n",
    "        return 'X'\n",
    "\n",
    "# Apply the function to each row to create the new column\n",
    "caaspp_df['aggregate_level'] = caaspp_df.apply(determine_aggregate_level, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['1', '3', '4', '8', '28', '31', '52', '74', '75', '76', '77', '78',\n",
       "       '79', '80', '128', '144', '160', '240'], dtype=object)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df['student_group_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['T', 'D', 'S'], dtype=object)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df['aggregate_level'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['3', '4', '5', '6', '7', '8', '11', '13'], dtype=object)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df['grade'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['4', '6', '9', '7', '10'], dtype=object)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df['type_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming caaspp_df is your DataFrame and it already contains the 'type_id' column\n",
    "\n",
    "# Function to determine charter_yn based on type_id\n",
    "def get_charter_yn(type_id):\n",
    "    if pd.isnull(type_id) or type_id == '':\n",
    "        return 'NV'\n",
    "    mapping = {\n",
    "        '4': 'All',\n",
    "        '6': 'All',\n",
    "        '7': 'No',\n",
    "        '9': 'Yes',\n",
    "        '10': 'Yes'\n",
    "    }\n",
    "    return mapping.get(type_id, 'Other')  # 'Other' is a default value if type_id is not in the mapping\n",
    "\n",
    "# Apply the function to each row in the 'type_id' column to create the new 'charter_yn' column\n",
    "caaspp_df['charter_yn'] = caaspp_df['type_id'].apply(get_charter_yn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['All', 'Yes', 'No'], dtype=object)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df.charter_yn.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>charter_yn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2960990</td>\n",
       "      <td>2957491</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>420238</td>\n",
       "      <td>410559</td>\n",
       "      <td>2409.8</td>\n",
       "      <td>23.40</td>\n",
       "      <td>19.56</td>\n",
       "      <td>42.96</td>\n",
       "      <td>22.40</td>\n",
       "      <td>34.64</td>\n",
       "      <td>410290</td>\n",
       "      <td>17.47</td>\n",
       "      <td>57.69</td>\n",
       "      <td>24.84</td>\n",
       "      <td>16.18</td>\n",
       "      <td>50.24</td>\n",
       "      <td>33.58</td>\n",
       "      <td>11.75</td>\n",
       "      <td>70.96</td>\n",
       "      <td>17.29</td>\n",
       "      <td>16.07</td>\n",
       "      <td>60.00</td>\n",
       "      <td>23.94</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1512977</td>\n",
       "      <td>1511175</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>214436</td>\n",
       "      <td>209138</td>\n",
       "      <td>2402.4</td>\n",
       "      <td>21.27</td>\n",
       "      <td>18.77</td>\n",
       "      <td>40.04</td>\n",
       "      <td>22.45</td>\n",
       "      <td>37.51</td>\n",
       "      <td>208998</td>\n",
       "      <td>15.88</td>\n",
       "      <td>57.23</td>\n",
       "      <td>26.89</td>\n",
       "      <td>13.96</td>\n",
       "      <td>49.35</td>\n",
       "      <td>36.69</td>\n",
       "      <td>11.47</td>\n",
       "      <td>70.55</td>\n",
       "      <td>17.99</td>\n",
       "      <td>15.20</td>\n",
       "      <td>59.33</td>\n",
       "      <td>25.47</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1445927</td>\n",
       "      <td>1444241</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>205715</td>\n",
       "      <td>201341</td>\n",
       "      <td>2417.5</td>\n",
       "      <td>25.60</td>\n",
       "      <td>20.39</td>\n",
       "      <td>45.99</td>\n",
       "      <td>22.34</td>\n",
       "      <td>31.67</td>\n",
       "      <td>201212</td>\n",
       "      <td>19.12</td>\n",
       "      <td>58.17</td>\n",
       "      <td>22.71</td>\n",
       "      <td>18.48</td>\n",
       "      <td>51.16</td>\n",
       "      <td>30.36</td>\n",
       "      <td>12.04</td>\n",
       "      <td>71.39</td>\n",
       "      <td>16.57</td>\n",
       "      <td>16.96</td>\n",
       "      <td>60.69</td>\n",
       "      <td>22.35</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>8</td>\n",
       "      <td>B</td>\n",
       "      <td>557974</td>\n",
       "      <td>557303</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>24566</td>\n",
       "      <td>24451</td>\n",
       "      <td>2474.8</td>\n",
       "      <td>43.01</td>\n",
       "      <td>29.94</td>\n",
       "      <td>72.96</td>\n",
       "      <td>19.26</td>\n",
       "      <td>7.78</td>\n",
       "      <td>24435</td>\n",
       "      <td>29.83</td>\n",
       "      <td>63.91</td>\n",
       "      <td>6.26</td>\n",
       "      <td>29.77</td>\n",
       "      <td>60.20</td>\n",
       "      <td>10.02</td>\n",
       "      <td>18.18</td>\n",
       "      <td>74.83</td>\n",
       "      <td>6.98</td>\n",
       "      <td>28.87</td>\n",
       "      <td>64.16</td>\n",
       "      <td>6.97</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>28</td>\n",
       "      <td>B</td>\n",
       "      <td>22322</td>\n",
       "      <td>22290</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3265</td>\n",
       "      <td>3165</td>\n",
       "      <td>2344.5</td>\n",
       "      <td>7.30</td>\n",
       "      <td>10.30</td>\n",
       "      <td>17.60</td>\n",
       "      <td>21.04</td>\n",
       "      <td>61.36</td>\n",
       "      <td>3165</td>\n",
       "      <td>5.81</td>\n",
       "      <td>48.34</td>\n",
       "      <td>45.85</td>\n",
       "      <td>4.84</td>\n",
       "      <td>37.61</td>\n",
       "      <td>57.55</td>\n",
       "      <td>4.96</td>\n",
       "      <td>65.91</td>\n",
       "      <td>29.13</td>\n",
       "      <td>6.32</td>\n",
       "      <td>51.53</td>\n",
       "      <td>42.15</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>128</td>\n",
       "      <td>B</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.76</td>\n",
       "      <td>88.24</td>\n",
       "      <td>17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.76</td>\n",
       "      <td>88.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17.65</td>\n",
       "      <td>82.35</td>\n",
       "      <td>0.00</td>\n",
       "      <td>23.53</td>\n",
       "      <td>76.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2586.9</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2550.0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>20</td>\n",
       "      <td>15.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>45.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>20</td>\n",
       "      <td>15.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>45.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       county_code district_code school_code filler test_year  \\\n",
       "0               00         00000     0000000    NaN      2023   \n",
       "1               00         00000     0000000    NaN      2023   \n",
       "2               00         00000     0000000    NaN      2023   \n",
       "3               00         00000     0000000    NaN      2023   \n",
       "4               00         00000     0000000    NaN      2023   \n",
       "...            ...           ...         ...    ...       ...   \n",
       "524547          58         72769     0000000    NaN      2018   \n",
       "524548          58         72769     0000000    NaN      2018   \n",
       "524549          58         72769     0000000    NaN      2018   \n",
       "524550          58         72769     0000000    NaN      2018   \n",
       "524551          58         72769     0000000    NaN      2018   \n",
       "\n",
       "       student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                     1         B                    2960990   \n",
       "1                     3         B                    1512977   \n",
       "2                     4         B                    1445927   \n",
       "3                     8         B                     557974   \n",
       "4                    28         B                      22322   \n",
       "...                 ...       ...                        ...   \n",
       "524547              128         B                         17   \n",
       "524548              144         B                         20   \n",
       "524549              144         B                         20   \n",
       "524550              144         B                         20   \n",
       "524551              144         B                         20   \n",
       "\n",
       "       total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                       2957491     3       1            420238   \n",
       "1                       1511175     3       1            214436   \n",
       "2                       1444241     3       1            205715   \n",
       "3                        557303     3       1             24566   \n",
       "4                         22290     3       1              3265   \n",
       "...                         ...   ...     ...               ...   \n",
       "524547                       17    13       2                18   \n",
       "524548                       20    11       1                20   \n",
       "524549                       20    11       2                20   \n",
       "524550                       20    13       2                20   \n",
       "524551                       20    13       1                20   \n",
       "\n",
       "       students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0               410559           2409.8            23.40       19.56   \n",
       "1               209138           2402.4            21.27       18.77   \n",
       "2               201341           2417.5            25.60       20.39   \n",
       "3                24451           2474.8            43.01       29.94   \n",
       "4                 3165           2344.5             7.30       10.30   \n",
       "...                ...              ...              ...         ...   \n",
       "524547              17              NaN             0.00        0.00   \n",
       "524548              20           2586.9            30.00       20.00   \n",
       "524549              20           2550.0            10.00       20.00   \n",
       "524550              20              NaN            10.00       20.00   \n",
       "524551              20              NaN            30.00       20.00   \n",
       "\n",
       "       pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                  42.96              22.40           34.64   \n",
       "1                  40.04              22.45           37.51   \n",
       "2                  45.99              22.34           31.67   \n",
       "3                  72.96              19.26            7.78   \n",
       "4                  17.60              21.04           61.36   \n",
       "...                  ...                ...             ...   \n",
       "524547              0.00              11.76           88.24   \n",
       "524548             50.00              25.00           25.00   \n",
       "524549             30.00              35.00           35.00   \n",
       "524550             30.00              35.00           35.00   \n",
       "524551             50.00              25.00           25.00   \n",
       "\n",
       "       students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                    410290                17.47               57.69   \n",
       "1                    208998                15.88               57.23   \n",
       "2                    201212                19.12               58.17   \n",
       "3                     24435                29.83               63.91   \n",
       "4                      3165                 5.81               48.34   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                   17                 0.00               11.76   \n",
       "524548                   20                45.00               30.00   \n",
       "524549                   20                15.00               40.00   \n",
       "524550                   20                15.00               40.00   \n",
       "524551                   20                45.00               30.00   \n",
       "\n",
       "       area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                     24.84                16.18               50.24   \n",
       "1                     26.89                13.96               49.35   \n",
       "2                     22.71                18.48               51.16   \n",
       "3                      6.26                29.77               60.20   \n",
       "4                     45.85                 4.84               37.61   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                88.24                 0.00               17.65   \n",
       "524548                25.00                35.00               30.00   \n",
       "524549                45.00                25.00               35.00   \n",
       "524550                45.00                25.00               35.00   \n",
       "524551                25.00                35.00               30.00   \n",
       "\n",
       "       area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                     33.58                11.75               70.96   \n",
       "1                     36.69                11.47               70.55   \n",
       "2                     30.36                12.04               71.39   \n",
       "3                     10.02                18.18               74.83   \n",
       "4                     57.55                 4.96               65.91   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                82.35                 0.00               23.53   \n",
       "524548                35.00                25.00               65.00   \n",
       "524549                40.00                25.00               35.00   \n",
       "524550                40.00                25.00               35.00   \n",
       "524551                35.00                25.00               65.00   \n",
       "\n",
       "       area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                     17.29                16.07               60.00   \n",
       "1                     17.99                15.20               59.33   \n",
       "2                     16.57                16.96               60.69   \n",
       "3                      6.98                28.87               64.16   \n",
       "4                     29.13                 6.32               51.53   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                76.47                 0.00                0.00   \n",
       "524548                10.00                30.00               35.00   \n",
       "524549                40.00                 0.00                0.00   \n",
       "524550                40.00                 0.00                0.00   \n",
       "524551                10.00                30.00               35.00   \n",
       "\n",
       "       area_4_pct_below_std type_id is_wcc_entity aggregate_level charter_yn  \n",
       "0                     23.94       4         False               T        All  \n",
       "1                     25.47       4         False               T        All  \n",
       "2                     22.35       4         False               T        All  \n",
       "3                      6.97       4         False               T        All  \n",
       "4                     42.15       4         False               T        All  \n",
       "...                     ...     ...           ...             ...        ...  \n",
       "524547                 0.00       6         False               D        All  \n",
       "524548                35.00       6         False               D        All  \n",
       "524549                 0.00       6         False               D        All  \n",
       "524550                 0.00       6         False               D        All  \n",
       "524551                35.00       6         False               D        All  \n",
       "\n",
       "[524552 rows x 36 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of columns to clean\n",
    "columns_to_clean = [\n",
    "    'area_1_pct_above_std', 'area_1_pct_near_std', 'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std', 'area_2_pct_near_std', 'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std', 'area_3_pct_near_std', 'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std', 'area_4_pct_near_std', 'area_4_pct_below_std'\n",
    "]\n",
    "\n",
    "# Replace any value containing '*' with an empty string (or another placeholder if needed)\n",
    "for column in columns_to_clean:\n",
    "    caaspp_df[column] = caaspp_df[column].replace(to_replace=r'.*\\*.*', value='', regex=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>charter_yn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2960990</td>\n",
       "      <td>2957491</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>420238</td>\n",
       "      <td>410559</td>\n",
       "      <td>2409.8</td>\n",
       "      <td>23.40</td>\n",
       "      <td>19.56</td>\n",
       "      <td>42.96</td>\n",
       "      <td>22.40</td>\n",
       "      <td>34.64</td>\n",
       "      <td>410290</td>\n",
       "      <td>17.47</td>\n",
       "      <td>57.69</td>\n",
       "      <td>24.84</td>\n",
       "      <td>16.18</td>\n",
       "      <td>50.24</td>\n",
       "      <td>33.58</td>\n",
       "      <td>11.75</td>\n",
       "      <td>70.96</td>\n",
       "      <td>17.29</td>\n",
       "      <td>16.07</td>\n",
       "      <td>60.00</td>\n",
       "      <td>23.94</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1512977</td>\n",
       "      <td>1511175</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>214436</td>\n",
       "      <td>209138</td>\n",
       "      <td>2402.4</td>\n",
       "      <td>21.27</td>\n",
       "      <td>18.77</td>\n",
       "      <td>40.04</td>\n",
       "      <td>22.45</td>\n",
       "      <td>37.51</td>\n",
       "      <td>208998</td>\n",
       "      <td>15.88</td>\n",
       "      <td>57.23</td>\n",
       "      <td>26.89</td>\n",
       "      <td>13.96</td>\n",
       "      <td>49.35</td>\n",
       "      <td>36.69</td>\n",
       "      <td>11.47</td>\n",
       "      <td>70.55</td>\n",
       "      <td>17.99</td>\n",
       "      <td>15.20</td>\n",
       "      <td>59.33</td>\n",
       "      <td>25.47</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1445927</td>\n",
       "      <td>1444241</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>205715</td>\n",
       "      <td>201341</td>\n",
       "      <td>2417.5</td>\n",
       "      <td>25.60</td>\n",
       "      <td>20.39</td>\n",
       "      <td>45.99</td>\n",
       "      <td>22.34</td>\n",
       "      <td>31.67</td>\n",
       "      <td>201212</td>\n",
       "      <td>19.12</td>\n",
       "      <td>58.17</td>\n",
       "      <td>22.71</td>\n",
       "      <td>18.48</td>\n",
       "      <td>51.16</td>\n",
       "      <td>30.36</td>\n",
       "      <td>12.04</td>\n",
       "      <td>71.39</td>\n",
       "      <td>16.57</td>\n",
       "      <td>16.96</td>\n",
       "      <td>60.69</td>\n",
       "      <td>22.35</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>8</td>\n",
       "      <td>B</td>\n",
       "      <td>557974</td>\n",
       "      <td>557303</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>24566</td>\n",
       "      <td>24451</td>\n",
       "      <td>2474.8</td>\n",
       "      <td>43.01</td>\n",
       "      <td>29.94</td>\n",
       "      <td>72.96</td>\n",
       "      <td>19.26</td>\n",
       "      <td>7.78</td>\n",
       "      <td>24435</td>\n",
       "      <td>29.83</td>\n",
       "      <td>63.91</td>\n",
       "      <td>6.26</td>\n",
       "      <td>29.77</td>\n",
       "      <td>60.20</td>\n",
       "      <td>10.02</td>\n",
       "      <td>18.18</td>\n",
       "      <td>74.83</td>\n",
       "      <td>6.98</td>\n",
       "      <td>28.87</td>\n",
       "      <td>64.16</td>\n",
       "      <td>6.97</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>28</td>\n",
       "      <td>B</td>\n",
       "      <td>22322</td>\n",
       "      <td>22290</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3265</td>\n",
       "      <td>3165</td>\n",
       "      <td>2344.5</td>\n",
       "      <td>7.30</td>\n",
       "      <td>10.30</td>\n",
       "      <td>17.60</td>\n",
       "      <td>21.04</td>\n",
       "      <td>61.36</td>\n",
       "      <td>3165</td>\n",
       "      <td>5.81</td>\n",
       "      <td>48.34</td>\n",
       "      <td>45.85</td>\n",
       "      <td>4.84</td>\n",
       "      <td>37.61</td>\n",
       "      <td>57.55</td>\n",
       "      <td>4.96</td>\n",
       "      <td>65.91</td>\n",
       "      <td>29.13</td>\n",
       "      <td>6.32</td>\n",
       "      <td>51.53</td>\n",
       "      <td>42.15</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>128</td>\n",
       "      <td>B</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.76</td>\n",
       "      <td>88.24</td>\n",
       "      <td>17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.76</td>\n",
       "      <td>88.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17.65</td>\n",
       "      <td>82.35</td>\n",
       "      <td>0.00</td>\n",
       "      <td>23.53</td>\n",
       "      <td>76.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2586.9</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2550.0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>20</td>\n",
       "      <td>15.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>45.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>20</td>\n",
       "      <td>15.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>45.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       county_code district_code school_code filler test_year  \\\n",
       "0               00         00000     0000000    NaN      2023   \n",
       "1               00         00000     0000000    NaN      2023   \n",
       "2               00         00000     0000000    NaN      2023   \n",
       "3               00         00000     0000000    NaN      2023   \n",
       "4               00         00000     0000000    NaN      2023   \n",
       "...            ...           ...         ...    ...       ...   \n",
       "524547          58         72769     0000000    NaN      2018   \n",
       "524548          58         72769     0000000    NaN      2018   \n",
       "524549          58         72769     0000000    NaN      2018   \n",
       "524550          58         72769     0000000    NaN      2018   \n",
       "524551          58         72769     0000000    NaN      2018   \n",
       "\n",
       "       student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                     1         B                    2960990   \n",
       "1                     3         B                    1512977   \n",
       "2                     4         B                    1445927   \n",
       "3                     8         B                     557974   \n",
       "4                    28         B                      22322   \n",
       "...                 ...       ...                        ...   \n",
       "524547              128         B                         17   \n",
       "524548              144         B                         20   \n",
       "524549              144         B                         20   \n",
       "524550              144         B                         20   \n",
       "524551              144         B                         20   \n",
       "\n",
       "       total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                       2957491     3       1            420238   \n",
       "1                       1511175     3       1            214436   \n",
       "2                       1444241     3       1            205715   \n",
       "3                        557303     3       1             24566   \n",
       "4                         22290     3       1              3265   \n",
       "...                         ...   ...     ...               ...   \n",
       "524547                       17    13       2                18   \n",
       "524548                       20    11       1                20   \n",
       "524549                       20    11       2                20   \n",
       "524550                       20    13       2                20   \n",
       "524551                       20    13       1                20   \n",
       "\n",
       "       students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0               410559           2409.8            23.40       19.56   \n",
       "1               209138           2402.4            21.27       18.77   \n",
       "2               201341           2417.5            25.60       20.39   \n",
       "3                24451           2474.8            43.01       29.94   \n",
       "4                 3165           2344.5             7.30       10.30   \n",
       "...                ...              ...              ...         ...   \n",
       "524547              17              NaN             0.00        0.00   \n",
       "524548              20           2586.9            30.00       20.00   \n",
       "524549              20           2550.0            10.00       20.00   \n",
       "524550              20              NaN            10.00       20.00   \n",
       "524551              20              NaN            30.00       20.00   \n",
       "\n",
       "       pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                  42.96              22.40           34.64   \n",
       "1                  40.04              22.45           37.51   \n",
       "2                  45.99              22.34           31.67   \n",
       "3                  72.96              19.26            7.78   \n",
       "4                  17.60              21.04           61.36   \n",
       "...                  ...                ...             ...   \n",
       "524547              0.00              11.76           88.24   \n",
       "524548             50.00              25.00           25.00   \n",
       "524549             30.00              35.00           35.00   \n",
       "524550             30.00              35.00           35.00   \n",
       "524551             50.00              25.00           25.00   \n",
       "\n",
       "       students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                    410290                17.47               57.69   \n",
       "1                    208998                15.88               57.23   \n",
       "2                    201212                19.12               58.17   \n",
       "3                     24435                29.83               63.91   \n",
       "4                      3165                 5.81               48.34   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                   17                 0.00               11.76   \n",
       "524548                   20                45.00               30.00   \n",
       "524549                   20                15.00               40.00   \n",
       "524550                   20                15.00               40.00   \n",
       "524551                   20                45.00               30.00   \n",
       "\n",
       "       area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                     24.84                16.18               50.24   \n",
       "1                     26.89                13.96               49.35   \n",
       "2                     22.71                18.48               51.16   \n",
       "3                      6.26                29.77               60.20   \n",
       "4                     45.85                 4.84               37.61   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                88.24                 0.00               17.65   \n",
       "524548                25.00                35.00               30.00   \n",
       "524549                45.00                25.00               35.00   \n",
       "524550                45.00                25.00               35.00   \n",
       "524551                25.00                35.00               30.00   \n",
       "\n",
       "       area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                     33.58                11.75               70.96   \n",
       "1                     36.69                11.47               70.55   \n",
       "2                     30.36                12.04               71.39   \n",
       "3                     10.02                18.18               74.83   \n",
       "4                     57.55                 4.96               65.91   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                82.35                 0.00               23.53   \n",
       "524548                35.00                25.00               65.00   \n",
       "524549                40.00                25.00               35.00   \n",
       "524550                40.00                25.00               35.00   \n",
       "524551                35.00                25.00               65.00   \n",
       "\n",
       "       area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                     17.29                16.07               60.00   \n",
       "1                     17.99                15.20               59.33   \n",
       "2                     16.57                16.96               60.69   \n",
       "3                      6.98                28.87               64.16   \n",
       "4                     29.13                 6.32               51.53   \n",
       "...                     ...                  ...                 ...   \n",
       "524547                76.47                 0.00                0.00   \n",
       "524548                10.00                30.00               35.00   \n",
       "524549                40.00                 0.00                0.00   \n",
       "524550                40.00                 0.00                0.00   \n",
       "524551                10.00                30.00               35.00   \n",
       "\n",
       "       area_4_pct_below_std type_id is_wcc_entity aggregate_level charter_yn  \n",
       "0                     23.94       4         False               T        All  \n",
       "1                     25.47       4         False               T        All  \n",
       "2                     22.35       4         False               T        All  \n",
       "3                      6.97       4         False               T        All  \n",
       "4                     42.15       4         False               T        All  \n",
       "...                     ...     ...           ...             ...        ...  \n",
       "524547                 0.00       6         False               D        All  \n",
       "524548                35.00       6         False               D        All  \n",
       "524549                 0.00       6         False               D        All  \n",
       "524550                 0.00       6         False               D        All  \n",
       "524551                35.00       6         False               D        All  \n",
       "\n",
       "[524552 rows x 36 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan], dtype=object)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df['filler'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "caaspp_df['pct_std_exceeded'] = (caaspp_df['pct_std_exceeded'].astype(float) / 100)\n",
    "caaspp_df['pct_std_met'] = (caaspp_df['pct_std_met'].astype(float) / 100)\n",
    "caaspp_df['pct_std_met_above'] = (caaspp_df['pct_std_met_above'].astype(float) / 100)\n",
    "caaspp_df['pct_std_nearly_met'] = (caaspp_df['pct_std_nearly_met'].astype(float) / 100)\n",
    "caaspp_df['pct_std_not_met'] = (caaspp_df['pct_std_not_met'].astype(float) / 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of columns you want to convert and divide by 100\n",
    "columns_to_convert = [\n",
    "    'area_1_pct_above_std', 'area_1_pct_near_std', 'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std', 'area_2_pct_near_std', 'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std', 'area_3_pct_near_std', 'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std', 'area_4_pct_near_std', 'area_4_pct_below_std'\n",
    "]\n",
    "\n",
    "# Convert columns to numeric, coercing errors to NaN, and divide by 100\n",
    "for column in columns_to_convert:\n",
    "    caaspp_df[column] = pd.to_numeric(caaspp_df[column], errors='coerce') / 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>charter_yn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2960990</td>\n",
       "      <td>2957491</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>420238</td>\n",
       "      <td>410559</td>\n",
       "      <td>2409.8</td>\n",
       "      <td>0.2340</td>\n",
       "      <td>0.1956</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>0.2240</td>\n",
       "      <td>0.3464</td>\n",
       "      <td>410290</td>\n",
       "      <td>0.1747</td>\n",
       "      <td>0.5769</td>\n",
       "      <td>0.2484</td>\n",
       "      <td>0.1618</td>\n",
       "      <td>0.5024</td>\n",
       "      <td>0.3358</td>\n",
       "      <td>0.1175</td>\n",
       "      <td>0.7096</td>\n",
       "      <td>0.1729</td>\n",
       "      <td>0.1607</td>\n",
       "      <td>0.6000</td>\n",
       "      <td>0.2394</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1512977</td>\n",
       "      <td>1511175</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>214436</td>\n",
       "      <td>209138</td>\n",
       "      <td>2402.4</td>\n",
       "      <td>0.2127</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.2245</td>\n",
       "      <td>0.3751</td>\n",
       "      <td>208998</td>\n",
       "      <td>0.1588</td>\n",
       "      <td>0.5723</td>\n",
       "      <td>0.2689</td>\n",
       "      <td>0.1396</td>\n",
       "      <td>0.4935</td>\n",
       "      <td>0.3669</td>\n",
       "      <td>0.1147</td>\n",
       "      <td>0.7055</td>\n",
       "      <td>0.1799</td>\n",
       "      <td>0.1520</td>\n",
       "      <td>0.5933</td>\n",
       "      <td>0.2547</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1445927</td>\n",
       "      <td>1444241</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>205715</td>\n",
       "      <td>201341</td>\n",
       "      <td>2417.5</td>\n",
       "      <td>0.2560</td>\n",
       "      <td>0.2039</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.2234</td>\n",
       "      <td>0.3167</td>\n",
       "      <td>201212</td>\n",
       "      <td>0.1912</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>0.2271</td>\n",
       "      <td>0.1848</td>\n",
       "      <td>0.5116</td>\n",
       "      <td>0.3036</td>\n",
       "      <td>0.1204</td>\n",
       "      <td>0.7139</td>\n",
       "      <td>0.1657</td>\n",
       "      <td>0.1696</td>\n",
       "      <td>0.6069</td>\n",
       "      <td>0.2235</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>8</td>\n",
       "      <td>B</td>\n",
       "      <td>557974</td>\n",
       "      <td>557303</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>24566</td>\n",
       "      <td>24451</td>\n",
       "      <td>2474.8</td>\n",
       "      <td>0.4301</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.1926</td>\n",
       "      <td>0.0778</td>\n",
       "      <td>24435</td>\n",
       "      <td>0.2983</td>\n",
       "      <td>0.6391</td>\n",
       "      <td>0.0626</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.6020</td>\n",
       "      <td>0.1002</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0.7483</td>\n",
       "      <td>0.0698</td>\n",
       "      <td>0.2887</td>\n",
       "      <td>0.6416</td>\n",
       "      <td>0.0697</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>28</td>\n",
       "      <td>B</td>\n",
       "      <td>22322</td>\n",
       "      <td>22290</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3265</td>\n",
       "      <td>3165</td>\n",
       "      <td>2344.5</td>\n",
       "      <td>0.0730</td>\n",
       "      <td>0.1030</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.2104</td>\n",
       "      <td>0.6136</td>\n",
       "      <td>3165</td>\n",
       "      <td>0.0581</td>\n",
       "      <td>0.4834</td>\n",
       "      <td>0.4585</td>\n",
       "      <td>0.0484</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.5755</td>\n",
       "      <td>0.0496</td>\n",
       "      <td>0.6591</td>\n",
       "      <td>0.2913</td>\n",
       "      <td>0.0632</td>\n",
       "      <td>0.5153</td>\n",
       "      <td>0.4215</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>128</td>\n",
       "      <td>B</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>17</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1765</td>\n",
       "      <td>0.8235</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2353</td>\n",
       "      <td>0.7647</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2586.9</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2550.0</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>D</td>\n",
       "      <td>All</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       county_code district_code school_code filler test_year  \\\n",
       "0               00         00000     0000000    NaN      2023   \n",
       "1               00         00000     0000000    NaN      2023   \n",
       "2               00         00000     0000000    NaN      2023   \n",
       "3               00         00000     0000000    NaN      2023   \n",
       "4               00         00000     0000000    NaN      2023   \n",
       "...            ...           ...         ...    ...       ...   \n",
       "524547          58         72769     0000000    NaN      2018   \n",
       "524548          58         72769     0000000    NaN      2018   \n",
       "524549          58         72769     0000000    NaN      2018   \n",
       "524550          58         72769     0000000    NaN      2018   \n",
       "524551          58         72769     0000000    NaN      2018   \n",
       "\n",
       "       student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                     1         B                    2960990   \n",
       "1                     3         B                    1512977   \n",
       "2                     4         B                    1445927   \n",
       "3                     8         B                     557974   \n",
       "4                    28         B                      22322   \n",
       "...                 ...       ...                        ...   \n",
       "524547              128         B                         17   \n",
       "524548              144         B                         20   \n",
       "524549              144         B                         20   \n",
       "524550              144         B                         20   \n",
       "524551              144         B                         20   \n",
       "\n",
       "       total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                       2957491     3       1            420238   \n",
       "1                       1511175     3       1            214436   \n",
       "2                       1444241     3       1            205715   \n",
       "3                        557303     3       1             24566   \n",
       "4                         22290     3       1              3265   \n",
       "...                         ...   ...     ...               ...   \n",
       "524547                       17    13       2                18   \n",
       "524548                       20    11       1                20   \n",
       "524549                       20    11       2                20   \n",
       "524550                       20    13       2                20   \n",
       "524551                       20    13       1                20   \n",
       "\n",
       "       students_tested mean_scale_score  pct_std_exceeded  pct_std_met  \\\n",
       "0               410559           2409.8            0.2340       0.1956   \n",
       "1               209138           2402.4            0.2127       0.1877   \n",
       "2               201341           2417.5            0.2560       0.2039   \n",
       "3                24451           2474.8            0.4301       0.2994   \n",
       "4                 3165           2344.5            0.0730       0.1030   \n",
       "...                ...              ...               ...          ...   \n",
       "524547              17              NaN            0.0000       0.0000   \n",
       "524548              20           2586.9            0.3000       0.2000   \n",
       "524549              20           2550.0            0.1000       0.2000   \n",
       "524550              20              NaN            0.1000       0.2000   \n",
       "524551              20              NaN            0.3000       0.2000   \n",
       "\n",
       "        pct_std_met_above  pct_std_nearly_met  pct_std_not_met  \\\n",
       "0                  0.4296              0.2240           0.3464   \n",
       "1                  0.4004              0.2245           0.3751   \n",
       "2                  0.4599              0.2234           0.3167   \n",
       "3                  0.7296              0.1926           0.0778   \n",
       "4                  0.1760              0.2104           0.6136   \n",
       "...                   ...                 ...              ...   \n",
       "524547             0.0000              0.1176           0.8824   \n",
       "524548             0.5000              0.2500           0.2500   \n",
       "524549             0.3000              0.3500           0.3500   \n",
       "524550             0.3000              0.3500           0.3500   \n",
       "524551             0.5000              0.2500           0.2500   \n",
       "\n",
       "       students_with_scores  area_1_pct_above_std  area_1_pct_near_std  \\\n",
       "0                    410290                0.1747               0.5769   \n",
       "1                    208998                0.1588               0.5723   \n",
       "2                    201212                0.1912               0.5817   \n",
       "3                     24435                0.2983               0.6391   \n",
       "4                      3165                0.0581               0.4834   \n",
       "...                     ...                   ...                  ...   \n",
       "524547                   17                0.0000               0.1176   \n",
       "524548                   20                0.4500               0.3000   \n",
       "524549                   20                0.1500               0.4000   \n",
       "524550                   20                0.1500               0.4000   \n",
       "524551                   20                0.4500               0.3000   \n",
       "\n",
       "        area_1_pct_below_std  area_2_pct_above_std  area_2_pct_near_std  \\\n",
       "0                     0.2484                0.1618               0.5024   \n",
       "1                     0.2689                0.1396               0.4935   \n",
       "2                     0.2271                0.1848               0.5116   \n",
       "3                     0.0626                0.2977               0.6020   \n",
       "4                     0.4585                0.0484               0.3761   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.8824                0.0000               0.1765   \n",
       "524548                0.2500                0.3500               0.3000   \n",
       "524549                0.4500                0.2500               0.3500   \n",
       "524550                0.4500                0.2500               0.3500   \n",
       "524551                0.2500                0.3500               0.3000   \n",
       "\n",
       "        area_2_pct_below_std  area_3_pct_above_std  area_3_pct_near_std  \\\n",
       "0                     0.3358                0.1175               0.7096   \n",
       "1                     0.3669                0.1147               0.7055   \n",
       "2                     0.3036                0.1204               0.7139   \n",
       "3                     0.1002                0.1818               0.7483   \n",
       "4                     0.5755                0.0496               0.6591   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.8235                0.0000               0.2353   \n",
       "524548                0.3500                0.2500               0.6500   \n",
       "524549                0.4000                0.2500               0.3500   \n",
       "524550                0.4000                0.2500               0.3500   \n",
       "524551                0.3500                0.2500               0.6500   \n",
       "\n",
       "        area_3_pct_below_std  area_4_pct_above_std  area_4_pct_near_std  \\\n",
       "0                     0.1729                0.1607               0.6000   \n",
       "1                     0.1799                0.1520               0.5933   \n",
       "2                     0.1657                0.1696               0.6069   \n",
       "3                     0.0698                0.2887               0.6416   \n",
       "4                     0.2913                0.0632               0.5153   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.7647                0.0000               0.0000   \n",
       "524548                0.1000                0.3000               0.3500   \n",
       "524549                0.4000                0.0000               0.0000   \n",
       "524550                0.4000                0.0000               0.0000   \n",
       "524551                0.1000                0.3000               0.3500   \n",
       "\n",
       "        area_4_pct_below_std type_id is_wcc_entity aggregate_level charter_yn  \n",
       "0                     0.2394       4         False               T        All  \n",
       "1                     0.2547       4         False               T        All  \n",
       "2                     0.2235       4         False               T        All  \n",
       "3                     0.0697       4         False               T        All  \n",
       "4                     0.4215       4         False               T        All  \n",
       "...                      ...     ...           ...             ...        ...  \n",
       "524547                0.0000       6         False               D        All  \n",
       "524548                0.3500       6         False               D        All  \n",
       "524549                0.0000       6         False               D        All  \n",
       "524550                0.0000       6         False               D        All  \n",
       "524551                0.3500       6         False               D        All  \n",
       "\n",
       "[524552 rows x 36 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "caaspp_df['students_with_scores_met_above'] = round(caaspp_df['pct_std_met_above'] * caaspp_df['students_with_scores'].astype(int)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "caaspp_df['pct_std_met_and_above'] = caaspp_df['pct_std_met_above']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>test_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>grade</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>students_with_scores_met_above</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_and_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>0.2340</td>\n",
       "      <td>0.1956</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>0.2240</td>\n",
       "      <td>0.3464</td>\n",
       "      <td>0.1747</td>\n",
       "      <td>0.5769</td>\n",
       "      <td>0.2484</td>\n",
       "      <td>0.1618</td>\n",
       "      <td>0.5024</td>\n",
       "      <td>0.3358</td>\n",
       "      <td>0.1175</td>\n",
       "      <td>0.7096</td>\n",
       "      <td>0.1729</td>\n",
       "      <td>0.1607</td>\n",
       "      <td>0.6000</td>\n",
       "      <td>0.2394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>208998</td>\n",
       "      <td>83683</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.2127</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.2245</td>\n",
       "      <td>0.3751</td>\n",
       "      <td>0.1588</td>\n",
       "      <td>0.5723</td>\n",
       "      <td>0.2689</td>\n",
       "      <td>0.1396</td>\n",
       "      <td>0.4935</td>\n",
       "      <td>0.3669</td>\n",
       "      <td>0.1147</td>\n",
       "      <td>0.7055</td>\n",
       "      <td>0.1799</td>\n",
       "      <td>0.1520</td>\n",
       "      <td>0.5933</td>\n",
       "      <td>0.2547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>201212</td>\n",
       "      <td>92537</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.2560</td>\n",
       "      <td>0.2039</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.2234</td>\n",
       "      <td>0.3167</td>\n",
       "      <td>0.1912</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>0.2271</td>\n",
       "      <td>0.1848</td>\n",
       "      <td>0.5116</td>\n",
       "      <td>0.3036</td>\n",
       "      <td>0.1204</td>\n",
       "      <td>0.7139</td>\n",
       "      <td>0.1657</td>\n",
       "      <td>0.1696</td>\n",
       "      <td>0.6069</td>\n",
       "      <td>0.2235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>24435</td>\n",
       "      <td>17828</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.4301</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.1926</td>\n",
       "      <td>0.0778</td>\n",
       "      <td>0.2983</td>\n",
       "      <td>0.6391</td>\n",
       "      <td>0.0626</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.6020</td>\n",
       "      <td>0.1002</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0.7483</td>\n",
       "      <td>0.0698</td>\n",
       "      <td>0.2887</td>\n",
       "      <td>0.6416</td>\n",
       "      <td>0.0697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>3165</td>\n",
       "      <td>557</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.0730</td>\n",
       "      <td>0.1030</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.2104</td>\n",
       "      <td>0.6136</td>\n",
       "      <td>0.0581</td>\n",
       "      <td>0.4834</td>\n",
       "      <td>0.4585</td>\n",
       "      <td>0.0484</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.5755</td>\n",
       "      <td>0.0496</td>\n",
       "      <td>0.6591</td>\n",
       "      <td>0.2913</td>\n",
       "      <td>0.0632</td>\n",
       "      <td>0.5153</td>\n",
       "      <td>0.4215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>128</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1765</td>\n",
       "      <td>0.8235</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2353</td>\n",
       "      <td>0.7647</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       test_year aggregate_level test_id county_code district_code  \\\n",
       "0           2023               T       1          00         00000   \n",
       "1           2023               T       1          00         00000   \n",
       "2           2023               T       1          00         00000   \n",
       "3           2023               T       1          00         00000   \n",
       "4           2023               T       1          00         00000   \n",
       "...          ...             ...     ...         ...           ...   \n",
       "524547      2018               D       2          58         72769   \n",
       "524548      2018               D       1          58         72769   \n",
       "524549      2018               D       2          58         72769   \n",
       "524550      2018               D       2          58         72769   \n",
       "524551      2018               D       1          58         72769   \n",
       "\n",
       "       school_code charter_yn student_group_id grade students_with_scores  \\\n",
       "0          0000000        All                1     3               410290   \n",
       "1          0000000        All                3     3               208998   \n",
       "2          0000000        All                4     3               201212   \n",
       "3          0000000        All                8     3                24435   \n",
       "4          0000000        All               28     3                 3165   \n",
       "...            ...        ...              ...   ...                  ...   \n",
       "524547     0000000        All              128    13                   17   \n",
       "524548     0000000        All              144    11                   20   \n",
       "524549     0000000        All              144    11                   20   \n",
       "524550     0000000        All              144    13                   20   \n",
       "524551     0000000        All              144    13                   20   \n",
       "\n",
       "        students_with_scores_met_above  pct_std_met_above  pct_std_exceeded  \\\n",
       "0                               176261             0.4296            0.2340   \n",
       "1                                83683             0.4004            0.2127   \n",
       "2                                92537             0.4599            0.2560   \n",
       "3                                17828             0.7296            0.4301   \n",
       "4                                  557             0.1760            0.0730   \n",
       "...                                ...                ...               ...   \n",
       "524547                               0             0.0000            0.0000   \n",
       "524548                              10             0.5000            0.3000   \n",
       "524549                               6             0.3000            0.1000   \n",
       "524550                               6             0.3000            0.1000   \n",
       "524551                              10             0.5000            0.3000   \n",
       "\n",
       "        pct_std_met  pct_std_met_and_above  pct_std_nearly_met  \\\n",
       "0            0.1956                 0.4296              0.2240   \n",
       "1            0.1877                 0.4004              0.2245   \n",
       "2            0.2039                 0.4599              0.2234   \n",
       "3            0.2994                 0.7296              0.1926   \n",
       "4            0.1030                 0.1760              0.2104   \n",
       "...             ...                    ...                 ...   \n",
       "524547       0.0000                 0.0000              0.1176   \n",
       "524548       0.2000                 0.5000              0.2500   \n",
       "524549       0.2000                 0.3000              0.3500   \n",
       "524550       0.2000                 0.3000              0.3500   \n",
       "524551       0.2000                 0.5000              0.2500   \n",
       "\n",
       "        pct_std_not_met  area_1_pct_above_std  area_1_pct_near_std  \\\n",
       "0                0.3464                0.1747               0.5769   \n",
       "1                0.3751                0.1588               0.5723   \n",
       "2                0.3167                0.1912               0.5817   \n",
       "3                0.0778                0.2983               0.6391   \n",
       "4                0.6136                0.0581               0.4834   \n",
       "...                 ...                   ...                  ...   \n",
       "524547           0.8824                0.0000               0.1176   \n",
       "524548           0.2500                0.4500               0.3000   \n",
       "524549           0.3500                0.1500               0.4000   \n",
       "524550           0.3500                0.1500               0.4000   \n",
       "524551           0.2500                0.4500               0.3000   \n",
       "\n",
       "        area_1_pct_below_std  area_2_pct_above_std  area_2_pct_near_std  \\\n",
       "0                     0.2484                0.1618               0.5024   \n",
       "1                     0.2689                0.1396               0.4935   \n",
       "2                     0.2271                0.1848               0.5116   \n",
       "3                     0.0626                0.2977               0.6020   \n",
       "4                     0.4585                0.0484               0.3761   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.8824                0.0000               0.1765   \n",
       "524548                0.2500                0.3500               0.3000   \n",
       "524549                0.4500                0.2500               0.3500   \n",
       "524550                0.4500                0.2500               0.3500   \n",
       "524551                0.2500                0.3500               0.3000   \n",
       "\n",
       "        area_2_pct_below_std  area_3_pct_above_std  area_3_pct_near_std  \\\n",
       "0                     0.3358                0.1175               0.7096   \n",
       "1                     0.3669                0.1147               0.7055   \n",
       "2                     0.3036                0.1204               0.7139   \n",
       "3                     0.1002                0.1818               0.7483   \n",
       "4                     0.5755                0.0496               0.6591   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.8235                0.0000               0.2353   \n",
       "524548                0.3500                0.2500               0.6500   \n",
       "524549                0.4000                0.2500               0.3500   \n",
       "524550                0.4000                0.2500               0.3500   \n",
       "524551                0.3500                0.2500               0.6500   \n",
       "\n",
       "        area_3_pct_below_std  area_4_pct_above_std  area_4_pct_near_std  \\\n",
       "0                     0.1729                0.1607               0.6000   \n",
       "1                     0.1799                0.1520               0.5933   \n",
       "2                     0.1657                0.1696               0.6069   \n",
       "3                     0.0698                0.2887               0.6416   \n",
       "4                     0.2913                0.0632               0.5153   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.7647                0.0000               0.0000   \n",
       "524548                0.1000                0.3000               0.3500   \n",
       "524549                0.4000                0.0000               0.0000   \n",
       "524550                0.4000                0.0000               0.0000   \n",
       "524551                0.1000                0.3000               0.3500   \n",
       "\n",
       "        area_4_pct_below_std  \n",
       "0                     0.2394  \n",
       "1                     0.2547  \n",
       "2                     0.2235  \n",
       "3                     0.0697  \n",
       "4                     0.4215  \n",
       "...                      ...  \n",
       "524547                0.0000  \n",
       "524548                0.3500  \n",
       "524549                0.0000  \n",
       "524550                0.0000  \n",
       "524551                0.3500  \n",
       "\n",
       "[524552 rows x 29 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the new column order\n",
    "new_order = [\n",
    "    'test_year',\n",
    "    'aggregate_level',\n",
    "    'test_id',\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'charter_yn',\n",
    "    'student_group_id',\n",
    "    'grade',\n",
    "    'students_with_scores',\n",
    "    'students_with_scores_met_above',\n",
    "    'pct_std_met_above',\n",
    "    \n",
    "    # ADD BELOW\n",
    "    'pct_std_exceeded',\n",
    "    'pct_std_met',\n",
    "    'pct_std_met_and_above',\n",
    "    'pct_std_nearly_met',\n",
    "    'pct_std_not_met',\n",
    "    \n",
    "    'area_1_pct_above_std', 'area_1_pct_near_std', 'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std', 'area_2_pct_near_std', 'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std', 'area_3_pct_near_std', 'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std', 'area_4_pct_near_std', 'area_4_pct_below_std'\n",
    "]\n",
    "caaspp_df = caaspp_df[new_order]\n",
    "caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>test_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>grade</th>\n",
       "      <th>total_all_students_with_scores</th>\n",
       "      <th>total_all_students_with_scores_met_above</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>3</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>4</td>\n",
       "      <td>415839</td>\n",
       "      <td>181846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>5</td>\n",
       "      <td>421672</td>\n",
       "      <td>196879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>6</td>\n",
       "      <td>425389</td>\n",
       "      <td>187852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>7</td>\n",
       "      <td>423896</td>\n",
       "      <td>201096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524405</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72751</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>13</td>\n",
       "      <td>816</td>\n",
       "      <td>325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524516</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>11</td>\n",
       "      <td>165</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524517</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>11</td>\n",
       "      <td>162</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524518</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>13</td>\n",
       "      <td>162</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524519</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>13</td>\n",
       "      <td>165</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>58266 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       test_year aggregate_level test_id county_code district_code  \\\n",
       "0           2023               T       1          00         00000   \n",
       "18          2023               T       1          00         00000   \n",
       "36          2023               T       1          00         00000   \n",
       "54          2023               T       1          00         00000   \n",
       "72          2023               T       1          00         00000   \n",
       "...          ...             ...     ...         ...           ...   \n",
       "524405      2018               D       2          58         72751   \n",
       "524516      2018               D       1          58         72769   \n",
       "524517      2018               D       2          58         72769   \n",
       "524518      2018               D       2          58         72769   \n",
       "524519      2018               D       1          58         72769   \n",
       "\n",
       "       school_code charter_yn grade total_all_students_with_scores  \\\n",
       "0          0000000        All     3                         410290   \n",
       "18         0000000        All     4                         415839   \n",
       "36         0000000        All     5                         421672   \n",
       "54         0000000        All     6                         425389   \n",
       "72         0000000        All     7                         423896   \n",
       "...            ...        ...   ...                            ...   \n",
       "524405     0000000        All    13                            816   \n",
       "524516     0000000        All    11                            165   \n",
       "524517     0000000        All    11                            162   \n",
       "524518     0000000        All    13                            162   \n",
       "524519     0000000        All    13                            165   \n",
       "\n",
       "        total_all_students_with_scores_met_above  \n",
       "0                                         176261  \n",
       "18                                        181846  \n",
       "36                                        196879  \n",
       "54                                        187852  \n",
       "72                                        201096  \n",
       "...                                          ...  \n",
       "524405                                       325  \n",
       "524516                                        85  \n",
       "524517                                        37  \n",
       "524518                                        37  \n",
       "524519                                        85  \n",
       "\n",
       "[58266 rows x 10 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_figures_casp_df = caaspp_df[caaspp_df['student_group_id'] == '1'][['test_year', 'aggregate_level', 'test_id', 'county_code', 'district_code', 'school_code', 'charter_yn', 'grade', 'students_with_scores', 'students_with_scores_met_above']]\n",
    "\n",
    "total_figures_casp_df = total_figures_casp_df.rename(columns={\n",
    "    'students_with_scores': 'total_all_students_with_scores',\n",
    "    'students_with_scores_met_above': 'total_all_students_with_scores_met_above'\n",
    "})\n",
    "\n",
    "\n",
    "total_figures_casp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>test_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>grade</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>students_with_scores_met_above</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_and_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>total_all_students_with_scores</th>\n",
       "      <th>total_all_students_with_scores_met_above</th>\n",
       "      <th>students_with_scores_diff</th>\n",
       "      <th>students_with_scores_met_above_diff</th>\n",
       "      <th>pct_std_met_above_non_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>0.2340</td>\n",
       "      <td>0.1956</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>0.2240</td>\n",
       "      <td>0.3464</td>\n",
       "      <td>0.1747</td>\n",
       "      <td>0.5769</td>\n",
       "      <td>0.2484</td>\n",
       "      <td>0.1618</td>\n",
       "      <td>0.5024</td>\n",
       "      <td>0.3358</td>\n",
       "      <td>0.1175</td>\n",
       "      <td>0.7096</td>\n",
       "      <td>0.1729</td>\n",
       "      <td>0.1607</td>\n",
       "      <td>0.6000</td>\n",
       "      <td>0.2394</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>208998</td>\n",
       "      <td>83683</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.2127</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.2245</td>\n",
       "      <td>0.3751</td>\n",
       "      <td>0.1588</td>\n",
       "      <td>0.5723</td>\n",
       "      <td>0.2689</td>\n",
       "      <td>0.1396</td>\n",
       "      <td>0.4935</td>\n",
       "      <td>0.3669</td>\n",
       "      <td>0.1147</td>\n",
       "      <td>0.7055</td>\n",
       "      <td>0.1799</td>\n",
       "      <td>0.1520</td>\n",
       "      <td>0.5933</td>\n",
       "      <td>0.2547</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>201292</td>\n",
       "      <td>92578</td>\n",
       "      <td>0.45992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>201212</td>\n",
       "      <td>92537</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.2560</td>\n",
       "      <td>0.2039</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.2234</td>\n",
       "      <td>0.3167</td>\n",
       "      <td>0.1912</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>0.2271</td>\n",
       "      <td>0.1848</td>\n",
       "      <td>0.5116</td>\n",
       "      <td>0.3036</td>\n",
       "      <td>0.1204</td>\n",
       "      <td>0.7139</td>\n",
       "      <td>0.1657</td>\n",
       "      <td>0.1696</td>\n",
       "      <td>0.6069</td>\n",
       "      <td>0.2235</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>209078</td>\n",
       "      <td>83724</td>\n",
       "      <td>0.40044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>24435</td>\n",
       "      <td>17828</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.4301</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.1926</td>\n",
       "      <td>0.0778</td>\n",
       "      <td>0.2983</td>\n",
       "      <td>0.6391</td>\n",
       "      <td>0.0626</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.6020</td>\n",
       "      <td>0.1002</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0.7483</td>\n",
       "      <td>0.0698</td>\n",
       "      <td>0.2887</td>\n",
       "      <td>0.6416</td>\n",
       "      <td>0.0697</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>385855</td>\n",
       "      <td>158433</td>\n",
       "      <td>0.41060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>3165</td>\n",
       "      <td>557</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.0730</td>\n",
       "      <td>0.1030</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.2104</td>\n",
       "      <td>0.6136</td>\n",
       "      <td>0.0581</td>\n",
       "      <td>0.4834</td>\n",
       "      <td>0.4585</td>\n",
       "      <td>0.0484</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.5755</td>\n",
       "      <td>0.0496</td>\n",
       "      <td>0.6591</td>\n",
       "      <td>0.2913</td>\n",
       "      <td>0.0632</td>\n",
       "      <td>0.5153</td>\n",
       "      <td>0.4215</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>407125</td>\n",
       "      <td>175704</td>\n",
       "      <td>0.43157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>128</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1765</td>\n",
       "      <td>0.8235</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2353</td>\n",
       "      <td>0.7647</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>162</td>\n",
       "      <td>37</td>\n",
       "      <td>145</td>\n",
       "      <td>37</td>\n",
       "      <td>0.25517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>165</td>\n",
       "      <td>85</td>\n",
       "      <td>145</td>\n",
       "      <td>75</td>\n",
       "      <td>0.51724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>162</td>\n",
       "      <td>37</td>\n",
       "      <td>142</td>\n",
       "      <td>31</td>\n",
       "      <td>0.21831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>162</td>\n",
       "      <td>37</td>\n",
       "      <td>142</td>\n",
       "      <td>31</td>\n",
       "      <td>0.21831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>165</td>\n",
       "      <td>85</td>\n",
       "      <td>145</td>\n",
       "      <td>75</td>\n",
       "      <td>0.51724</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       test_year aggregate_level test_id county_code district_code  \\\n",
       "0           2023               T       1          00         00000   \n",
       "1           2023               T       1          00         00000   \n",
       "2           2023               T       1          00         00000   \n",
       "3           2023               T       1          00         00000   \n",
       "4           2023               T       1          00         00000   \n",
       "...          ...             ...     ...         ...           ...   \n",
       "524547      2018               D       2          58         72769   \n",
       "524548      2018               D       1          58         72769   \n",
       "524549      2018               D       2          58         72769   \n",
       "524550      2018               D       2          58         72769   \n",
       "524551      2018               D       1          58         72769   \n",
       "\n",
       "       school_code charter_yn student_group_id grade students_with_scores  \\\n",
       "0          0000000        All                1     3               410290   \n",
       "1          0000000        All                3     3               208998   \n",
       "2          0000000        All                4     3               201212   \n",
       "3          0000000        All                8     3                24435   \n",
       "4          0000000        All               28     3                 3165   \n",
       "...            ...        ...              ...   ...                  ...   \n",
       "524547     0000000        All              128    13                   17   \n",
       "524548     0000000        All              144    11                   20   \n",
       "524549     0000000        All              144    11                   20   \n",
       "524550     0000000        All              144    13                   20   \n",
       "524551     0000000        All              144    13                   20   \n",
       "\n",
       "        students_with_scores_met_above  pct_std_met_above  pct_std_exceeded  \\\n",
       "0                               176261             0.4296            0.2340   \n",
       "1                                83683             0.4004            0.2127   \n",
       "2                                92537             0.4599            0.2560   \n",
       "3                                17828             0.7296            0.4301   \n",
       "4                                  557             0.1760            0.0730   \n",
       "...                                ...                ...               ...   \n",
       "524547                               0             0.0000            0.0000   \n",
       "524548                              10             0.5000            0.3000   \n",
       "524549                               6             0.3000            0.1000   \n",
       "524550                               6             0.3000            0.1000   \n",
       "524551                              10             0.5000            0.3000   \n",
       "\n",
       "        pct_std_met  pct_std_met_and_above  pct_std_nearly_met  \\\n",
       "0            0.1956                 0.4296              0.2240   \n",
       "1            0.1877                 0.4004              0.2245   \n",
       "2            0.2039                 0.4599              0.2234   \n",
       "3            0.2994                 0.7296              0.1926   \n",
       "4            0.1030                 0.1760              0.2104   \n",
       "...             ...                    ...                 ...   \n",
       "524547       0.0000                 0.0000              0.1176   \n",
       "524548       0.2000                 0.5000              0.2500   \n",
       "524549       0.2000                 0.3000              0.3500   \n",
       "524550       0.2000                 0.3000              0.3500   \n",
       "524551       0.2000                 0.5000              0.2500   \n",
       "\n",
       "        pct_std_not_met  area_1_pct_above_std  area_1_pct_near_std  \\\n",
       "0                0.3464                0.1747               0.5769   \n",
       "1                0.3751                0.1588               0.5723   \n",
       "2                0.3167                0.1912               0.5817   \n",
       "3                0.0778                0.2983               0.6391   \n",
       "4                0.6136                0.0581               0.4834   \n",
       "...                 ...                   ...                  ...   \n",
       "524547           0.8824                0.0000               0.1176   \n",
       "524548           0.2500                0.4500               0.3000   \n",
       "524549           0.3500                0.1500               0.4000   \n",
       "524550           0.3500                0.1500               0.4000   \n",
       "524551           0.2500                0.4500               0.3000   \n",
       "\n",
       "        area_1_pct_below_std  area_2_pct_above_std  area_2_pct_near_std  \\\n",
       "0                     0.2484                0.1618               0.5024   \n",
       "1                     0.2689                0.1396               0.4935   \n",
       "2                     0.2271                0.1848               0.5116   \n",
       "3                     0.0626                0.2977               0.6020   \n",
       "4                     0.4585                0.0484               0.3761   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.8824                0.0000               0.1765   \n",
       "524548                0.2500                0.3500               0.3000   \n",
       "524549                0.4500                0.2500               0.3500   \n",
       "524550                0.4500                0.2500               0.3500   \n",
       "524551                0.2500                0.3500               0.3000   \n",
       "\n",
       "        area_2_pct_below_std  area_3_pct_above_std  area_3_pct_near_std  \\\n",
       "0                     0.3358                0.1175               0.7096   \n",
       "1                     0.3669                0.1147               0.7055   \n",
       "2                     0.3036                0.1204               0.7139   \n",
       "3                     0.1002                0.1818               0.7483   \n",
       "4                     0.5755                0.0496               0.6591   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.8235                0.0000               0.2353   \n",
       "524548                0.3500                0.2500               0.6500   \n",
       "524549                0.4000                0.2500               0.3500   \n",
       "524550                0.4000                0.2500               0.3500   \n",
       "524551                0.3500                0.2500               0.6500   \n",
       "\n",
       "        area_3_pct_below_std  area_4_pct_above_std  area_4_pct_near_std  \\\n",
       "0                     0.1729                0.1607               0.6000   \n",
       "1                     0.1799                0.1520               0.5933   \n",
       "2                     0.1657                0.1696               0.6069   \n",
       "3                     0.0698                0.2887               0.6416   \n",
       "4                     0.2913                0.0632               0.5153   \n",
       "...                      ...                   ...                  ...   \n",
       "524547                0.7647                0.0000               0.0000   \n",
       "524548                0.1000                0.3000               0.3500   \n",
       "524549                0.4000                0.0000               0.0000   \n",
       "524550                0.4000                0.0000               0.0000   \n",
       "524551                0.1000                0.3000               0.3500   \n",
       "\n",
       "        area_4_pct_below_std total_all_students_with_scores  \\\n",
       "0                     0.2394                         410290   \n",
       "1                     0.2547                         410290   \n",
       "2                     0.2235                         410290   \n",
       "3                     0.0697                         410290   \n",
       "4                     0.4215                         410290   \n",
       "...                      ...                            ...   \n",
       "524547                0.0000                            162   \n",
       "524548                0.3500                            165   \n",
       "524549                0.0000                            162   \n",
       "524550                0.0000                            162   \n",
       "524551                0.3500                            165   \n",
       "\n",
       "        total_all_students_with_scores_met_above  students_with_scores_diff  \\\n",
       "0                                         176261                          0   \n",
       "1                                         176261                     201292   \n",
       "2                                         176261                     209078   \n",
       "3                                         176261                     385855   \n",
       "4                                         176261                     407125   \n",
       "...                                          ...                        ...   \n",
       "524547                                        37                        145   \n",
       "524548                                        85                        145   \n",
       "524549                                        37                        142   \n",
       "524550                                        37                        142   \n",
       "524551                                        85                        145   \n",
       "\n",
       "        students_with_scores_met_above_diff  pct_std_met_above_non_group  \n",
       "0                                         0                          NaN  \n",
       "1                                     92578                      0.45992  \n",
       "2                                     83724                      0.40044  \n",
       "3                                    158433                      0.41060  \n",
       "4                                    175704                      0.43157  \n",
       "...                                     ...                          ...  \n",
       "524547                                   37                      0.25517  \n",
       "524548                                   75                      0.51724  \n",
       "524549                                   31                      0.21831  \n",
       "524550                                   31                      0.21831  \n",
       "524551                                   75                      0.51724  \n",
       "\n",
       "[524552 rows x 34 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df = caaspp_df.merge(total_figures_casp_df, on=['test_year', 'aggregate_level', 'test_id', 'county_code', 'district_code', 'school_code', 'charter_yn', 'grade'])\n",
    "caaspp_df['students_with_scores_diff'] = caaspp_df['total_all_students_with_scores'].astype(int) - caaspp_df['students_with_scores'].astype(int)\n",
    "caaspp_df['students_with_scores_met_above_diff'] = caaspp_df['total_all_students_with_scores_met_above'].astype(int) - caaspp_df['students_with_scores_met_above'].astype(int)\n",
    "\n",
    "caaspp_df['pct_std_met_above_non_group'] = (caaspp_df['students_with_scores_met_above_diff'].astype(int) / caaspp_df['students_with_scores_diff'].astype(int) ).round(5)\n",
    "caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>test_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>grade</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>students_with_scores_met_above</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_met_above_non_group</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_and_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2340</td>\n",
       "      <td>0.1956</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>0.2240</td>\n",
       "      <td>0.3464</td>\n",
       "      <td>0.1747</td>\n",
       "      <td>0.5769</td>\n",
       "      <td>0.2484</td>\n",
       "      <td>0.1618</td>\n",
       "      <td>0.5024</td>\n",
       "      <td>0.3358</td>\n",
       "      <td>0.1175</td>\n",
       "      <td>0.7096</td>\n",
       "      <td>0.1729</td>\n",
       "      <td>0.1607</td>\n",
       "      <td>0.6000</td>\n",
       "      <td>0.2394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>208998</td>\n",
       "      <td>83683</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.45992</td>\n",
       "      <td>0.2127</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.2245</td>\n",
       "      <td>0.3751</td>\n",
       "      <td>0.1588</td>\n",
       "      <td>0.5723</td>\n",
       "      <td>0.2689</td>\n",
       "      <td>0.1396</td>\n",
       "      <td>0.4935</td>\n",
       "      <td>0.3669</td>\n",
       "      <td>0.1147</td>\n",
       "      <td>0.7055</td>\n",
       "      <td>0.1799</td>\n",
       "      <td>0.1520</td>\n",
       "      <td>0.5933</td>\n",
       "      <td>0.2547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>201212</td>\n",
       "      <td>92537</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.40044</td>\n",
       "      <td>0.2560</td>\n",
       "      <td>0.2039</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.2234</td>\n",
       "      <td>0.3167</td>\n",
       "      <td>0.1912</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>0.2271</td>\n",
       "      <td>0.1848</td>\n",
       "      <td>0.5116</td>\n",
       "      <td>0.3036</td>\n",
       "      <td>0.1204</td>\n",
       "      <td>0.7139</td>\n",
       "      <td>0.1657</td>\n",
       "      <td>0.1696</td>\n",
       "      <td>0.6069</td>\n",
       "      <td>0.2235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>24435</td>\n",
       "      <td>17828</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.41060</td>\n",
       "      <td>0.4301</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.1926</td>\n",
       "      <td>0.0778</td>\n",
       "      <td>0.2983</td>\n",
       "      <td>0.6391</td>\n",
       "      <td>0.0626</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.6020</td>\n",
       "      <td>0.1002</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0.7483</td>\n",
       "      <td>0.0698</td>\n",
       "      <td>0.2887</td>\n",
       "      <td>0.6416</td>\n",
       "      <td>0.0697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>1</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>3165</td>\n",
       "      <td>557</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.43157</td>\n",
       "      <td>0.0730</td>\n",
       "      <td>0.1030</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.2104</td>\n",
       "      <td>0.6136</td>\n",
       "      <td>0.0581</td>\n",
       "      <td>0.4834</td>\n",
       "      <td>0.4585</td>\n",
       "      <td>0.0484</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.5755</td>\n",
       "      <td>0.0496</td>\n",
       "      <td>0.6591</td>\n",
       "      <td>0.2913</td>\n",
       "      <td>0.0632</td>\n",
       "      <td>0.5153</td>\n",
       "      <td>0.4215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>128</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.25517</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1765</td>\n",
       "      <td>0.8235</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2353</td>\n",
       "      <td>0.7647</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.51724</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.21831</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.21831</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>144</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.51724</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       test_year aggregate_level test_id county_code district_code  \\\n",
       "0           2023               T       1          00         00000   \n",
       "1           2023               T       1          00         00000   \n",
       "2           2023               T       1          00         00000   \n",
       "3           2023               T       1          00         00000   \n",
       "4           2023               T       1          00         00000   \n",
       "...          ...             ...     ...         ...           ...   \n",
       "524547      2018               D       2          58         72769   \n",
       "524548      2018               D       1          58         72769   \n",
       "524549      2018               D       2          58         72769   \n",
       "524550      2018               D       2          58         72769   \n",
       "524551      2018               D       1          58         72769   \n",
       "\n",
       "       school_code charter_yn student_group_id grade students_with_scores  \\\n",
       "0          0000000        All                1     3               410290   \n",
       "1          0000000        All                3     3               208998   \n",
       "2          0000000        All                4     3               201212   \n",
       "3          0000000        All                8     3                24435   \n",
       "4          0000000        All               28     3                 3165   \n",
       "...            ...        ...              ...   ...                  ...   \n",
       "524547     0000000        All              128    13                   17   \n",
       "524548     0000000        All              144    11                   20   \n",
       "524549     0000000        All              144    11                   20   \n",
       "524550     0000000        All              144    13                   20   \n",
       "524551     0000000        All              144    13                   20   \n",
       "\n",
       "        students_with_scores_met_above  pct_std_met_above  \\\n",
       "0                               176261             0.4296   \n",
       "1                                83683             0.4004   \n",
       "2                                92537             0.4599   \n",
       "3                                17828             0.7296   \n",
       "4                                  557             0.1760   \n",
       "...                                ...                ...   \n",
       "524547                               0             0.0000   \n",
       "524548                              10             0.5000   \n",
       "524549                               6             0.3000   \n",
       "524550                               6             0.3000   \n",
       "524551                              10             0.5000   \n",
       "\n",
       "        pct_std_met_above_non_group  pct_std_exceeded  pct_std_met  \\\n",
       "0                               NaN            0.2340       0.1956   \n",
       "1                           0.45992            0.2127       0.1877   \n",
       "2                           0.40044            0.2560       0.2039   \n",
       "3                           0.41060            0.4301       0.2994   \n",
       "4                           0.43157            0.0730       0.1030   \n",
       "...                             ...               ...          ...   \n",
       "524547                      0.25517            0.0000       0.0000   \n",
       "524548                      0.51724            0.3000       0.2000   \n",
       "524549                      0.21831            0.1000       0.2000   \n",
       "524550                      0.21831            0.1000       0.2000   \n",
       "524551                      0.51724            0.3000       0.2000   \n",
       "\n",
       "        pct_std_met_and_above  pct_std_nearly_met  pct_std_not_met  \\\n",
       "0                      0.4296              0.2240           0.3464   \n",
       "1                      0.4004              0.2245           0.3751   \n",
       "2                      0.4599              0.2234           0.3167   \n",
       "3                      0.7296              0.1926           0.0778   \n",
       "4                      0.1760              0.2104           0.6136   \n",
       "...                       ...                 ...              ...   \n",
       "524547                 0.0000              0.1176           0.8824   \n",
       "524548                 0.5000              0.2500           0.2500   \n",
       "524549                 0.3000              0.3500           0.3500   \n",
       "524550                 0.3000              0.3500           0.3500   \n",
       "524551                 0.5000              0.2500           0.2500   \n",
       "\n",
       "        area_1_pct_above_std  area_1_pct_near_std  area_1_pct_below_std  \\\n",
       "0                     0.1747               0.5769                0.2484   \n",
       "1                     0.1588               0.5723                0.2689   \n",
       "2                     0.1912               0.5817                0.2271   \n",
       "3                     0.2983               0.6391                0.0626   \n",
       "4                     0.0581               0.4834                0.4585   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.1176                0.8824   \n",
       "524548                0.4500               0.3000                0.2500   \n",
       "524549                0.1500               0.4000                0.4500   \n",
       "524550                0.1500               0.4000                0.4500   \n",
       "524551                0.4500               0.3000                0.2500   \n",
       "\n",
       "        area_2_pct_above_std  area_2_pct_near_std  area_2_pct_below_std  \\\n",
       "0                     0.1618               0.5024                0.3358   \n",
       "1                     0.1396               0.4935                0.3669   \n",
       "2                     0.1848               0.5116                0.3036   \n",
       "3                     0.2977               0.6020                0.1002   \n",
       "4                     0.0484               0.3761                0.5755   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.1765                0.8235   \n",
       "524548                0.3500               0.3000                0.3500   \n",
       "524549                0.2500               0.3500                0.4000   \n",
       "524550                0.2500               0.3500                0.4000   \n",
       "524551                0.3500               0.3000                0.3500   \n",
       "\n",
       "        area_3_pct_above_std  area_3_pct_near_std  area_3_pct_below_std  \\\n",
       "0                     0.1175               0.7096                0.1729   \n",
       "1                     0.1147               0.7055                0.1799   \n",
       "2                     0.1204               0.7139                0.1657   \n",
       "3                     0.1818               0.7483                0.0698   \n",
       "4                     0.0496               0.6591                0.2913   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.2353                0.7647   \n",
       "524548                0.2500               0.6500                0.1000   \n",
       "524549                0.2500               0.3500                0.4000   \n",
       "524550                0.2500               0.3500                0.4000   \n",
       "524551                0.2500               0.6500                0.1000   \n",
       "\n",
       "        area_4_pct_above_std  area_4_pct_near_std  area_4_pct_below_std  \n",
       "0                     0.1607               0.6000                0.2394  \n",
       "1                     0.1520               0.5933                0.2547  \n",
       "2                     0.1696               0.6069                0.2235  \n",
       "3                     0.2887               0.6416                0.0697  \n",
       "4                     0.0632               0.5153                0.4215  \n",
       "...                      ...                  ...                   ...  \n",
       "524547                0.0000               0.0000                0.0000  \n",
       "524548                0.3000               0.3500                0.3500  \n",
       "524549                0.0000               0.0000                0.0000  \n",
       "524550                0.0000               0.0000                0.0000  \n",
       "524551                0.3000               0.3500                0.3500  \n",
       "\n",
       "[524552 rows x 30 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify the new column order\n",
    "new_order = [\n",
    "    'test_year',\n",
    "    'aggregate_level',\n",
    "    'test_id',\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'charter_yn',\n",
    "\n",
    "\n",
    "    'student_group_id',\n",
    "    'grade',\n",
    "    'students_with_scores',\n",
    "    'students_with_scores_met_above',\n",
    "    'pct_std_met_above',\n",
    "    'pct_std_met_above_non_group',\n",
    "    \n",
    "    # ADD BELOW\n",
    "    'pct_std_exceeded',\n",
    "    'pct_std_met',\n",
    "    'pct_std_met_and_above',\n",
    "    'pct_std_nearly_met',\n",
    "    'pct_std_not_met',\n",
    "    \n",
    "    'area_1_pct_above_std', 'area_1_pct_near_std', 'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std', 'area_2_pct_near_std', 'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std', 'area_3_pct_near_std', 'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std', 'area_4_pct_near_std', 'area_4_pct_below_std'\n",
    "    \n",
    "]\n",
    "caaspp_df = caaspp_df[new_order]\n",
    "caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"Race and Ethnicity\tBlack or African American\t                    074\"\n",
    "# \"Race and Ethnicity\tAmerican Indian or Alaska Native\t            075\"\n",
    "# \"Race and Ethnicity\tAsian\t                                        076\"\n",
    "# \"Race and Ethnicity\tFilipino\t                                    077\"\n",
    "# Race and Ethnicity        Hispanic or Latino                          078\n",
    "\n",
    "# Race and Ethnicity        Native Hawaiian or Pacific Islander         079\"\n",
    "# \"Race and Ethnicity\tTwo or more races\t                            114\"\n",
    "# \"Race and Ethnicity\tWhite\t                                        080\"\n",
    "# \"Gender\tMale\t                                                    003\"\n",
    "# \"Gender\tFemale\t                                                    004\"\n",
    "\n",
    "\n",
    "# \"English-Language Fluency\tEL (English learner)\t                    160\"\n",
    "# \"Disability Status\tReported disabilities\t                        128\"\n",
    "# \"Economic Status\tSocioeconomically disadvantaged\t                    031\"\n",
    "# \"Migrant\tMigrant education\t                                        028\"\n",
    "# \"Foster Status\tFoster youth\t                                    240\"\n",
    "# \"Homeless Status\tHomeless\t                                        052\"\n",
    "\n",
    "# \"All Students\t                                                        001\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['1', '3', '4', '8', '28', '31', '52', '74', '75', '76', '77', '78',\n",
       "       '79', '80', '128', '144', '160', '240'], dtype=object)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df['student_group_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the mapping from string numbers to two-letter codes\n",
    "mapping = {\n",
    "    '1': 'TA',\n",
    "    '3': 'GM',\n",
    "    '4': 'GF',\n",
    "    '8': 'RFEP',\n",
    "    '28': 'SM',\n",
    "    '31': 'SS',\n",
    "    '52': 'SH',\n",
    "    '74': 'RB',\n",
    "    '75': 'RI',\n",
    "    '76': 'RA',\n",
    "    '77': 'RF',\n",
    "    '78': 'RH',\n",
    "    '79': 'RP',\n",
    "    '80': 'RW',\n",
    "    '128': 'SD',\n",
    "    '144': 'RT',\n",
    "    '160': 'SE',\n",
    "    '240': 'SF'\n",
    "}\n",
    "\n",
    "# Use .loc to ensure you're modifying the original DataFrame\n",
    "caaspp_df.loc[:, 'student_group_id'] = caaspp_df['student_group_id'].astype(str).replace(mapping)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/6t/n3rqwc2x06ngf2r8521g1nmc0000gn/T/ipykernel_20873/2776317192.py:9: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  caaspp_df.loc[:, 'test_id'] = caaspp_df['test_id'].astype(str).replace(metric_mapping)\n"
     ]
    }
   ],
   "source": [
    "# Define the mapping from metric_id\n",
    "metric_mapping = {\n",
    "    '1': 415,\n",
    "    '2': 416\n",
    "}\n",
    "\n",
    "\n",
    "# Use .loc to ensure you're modifying the original DataFrame\n",
    "caaspp_df.loc[:, 'test_id'] = caaspp_df['test_id'].astype(str).replace(metric_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['TA', 'GM', 'GF', 'RFEP', 'SM', 'SS', 'SH', 'RB', 'RI', 'RA', 'RF',\n",
       "       'RH', 'RP', 'RW', 'SD', 'RT', 'SE', 'SF'], dtype=object)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df['student_group_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>test_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>grade</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>students_with_scores_met_above</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_met_above_non_group</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_and_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>3</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2340</td>\n",
       "      <td>0.1956</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>0.2240</td>\n",
       "      <td>0.3464</td>\n",
       "      <td>0.1747</td>\n",
       "      <td>0.5769</td>\n",
       "      <td>0.2484</td>\n",
       "      <td>0.1618</td>\n",
       "      <td>0.5024</td>\n",
       "      <td>0.3358</td>\n",
       "      <td>0.1175</td>\n",
       "      <td>0.7096</td>\n",
       "      <td>0.1729</td>\n",
       "      <td>0.1607</td>\n",
       "      <td>0.6000</td>\n",
       "      <td>0.2394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>3</td>\n",
       "      <td>208998</td>\n",
       "      <td>83683</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.45992</td>\n",
       "      <td>0.2127</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.2245</td>\n",
       "      <td>0.3751</td>\n",
       "      <td>0.1588</td>\n",
       "      <td>0.5723</td>\n",
       "      <td>0.2689</td>\n",
       "      <td>0.1396</td>\n",
       "      <td>0.4935</td>\n",
       "      <td>0.3669</td>\n",
       "      <td>0.1147</td>\n",
       "      <td>0.7055</td>\n",
       "      <td>0.1799</td>\n",
       "      <td>0.1520</td>\n",
       "      <td>0.5933</td>\n",
       "      <td>0.2547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>3</td>\n",
       "      <td>201212</td>\n",
       "      <td>92537</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.40044</td>\n",
       "      <td>0.2560</td>\n",
       "      <td>0.2039</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.2234</td>\n",
       "      <td>0.3167</td>\n",
       "      <td>0.1912</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>0.2271</td>\n",
       "      <td>0.1848</td>\n",
       "      <td>0.5116</td>\n",
       "      <td>0.3036</td>\n",
       "      <td>0.1204</td>\n",
       "      <td>0.7139</td>\n",
       "      <td>0.1657</td>\n",
       "      <td>0.1696</td>\n",
       "      <td>0.6069</td>\n",
       "      <td>0.2235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RFEP</td>\n",
       "      <td>3</td>\n",
       "      <td>24435</td>\n",
       "      <td>17828</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.41060</td>\n",
       "      <td>0.4301</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.1926</td>\n",
       "      <td>0.0778</td>\n",
       "      <td>0.2983</td>\n",
       "      <td>0.6391</td>\n",
       "      <td>0.0626</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.6020</td>\n",
       "      <td>0.1002</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0.7483</td>\n",
       "      <td>0.0698</td>\n",
       "      <td>0.2887</td>\n",
       "      <td>0.6416</td>\n",
       "      <td>0.0697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SM</td>\n",
       "      <td>3</td>\n",
       "      <td>3165</td>\n",
       "      <td>557</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.43157</td>\n",
       "      <td>0.0730</td>\n",
       "      <td>0.1030</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.2104</td>\n",
       "      <td>0.6136</td>\n",
       "      <td>0.0581</td>\n",
       "      <td>0.4834</td>\n",
       "      <td>0.4585</td>\n",
       "      <td>0.0484</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.5755</td>\n",
       "      <td>0.0496</td>\n",
       "      <td>0.6591</td>\n",
       "      <td>0.2913</td>\n",
       "      <td>0.0632</td>\n",
       "      <td>0.5153</td>\n",
       "      <td>0.4215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SD</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.25517</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1765</td>\n",
       "      <td>0.8235</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2353</td>\n",
       "      <td>0.7647</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>415</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.51724</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.21831</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.21831</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>415</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.51724</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       test_year aggregate_level test_id county_code district_code  \\\n",
       "0           2023               T     415          00         00000   \n",
       "1           2023               T     415          00         00000   \n",
       "2           2023               T     415          00         00000   \n",
       "3           2023               T     415          00         00000   \n",
       "4           2023               T     415          00         00000   \n",
       "...          ...             ...     ...         ...           ...   \n",
       "524547      2018               D     416          58         72769   \n",
       "524548      2018               D     415          58         72769   \n",
       "524549      2018               D     416          58         72769   \n",
       "524550      2018               D     416          58         72769   \n",
       "524551      2018               D     415          58         72769   \n",
       "\n",
       "       school_code charter_yn student_group_id grade students_with_scores  \\\n",
       "0          0000000        All               TA     3               410290   \n",
       "1          0000000        All               GM     3               208998   \n",
       "2          0000000        All               GF     3               201212   \n",
       "3          0000000        All             RFEP     3                24435   \n",
       "4          0000000        All               SM     3                 3165   \n",
       "...            ...        ...              ...   ...                  ...   \n",
       "524547     0000000        All               SD    13                   17   \n",
       "524548     0000000        All               RT    11                   20   \n",
       "524549     0000000        All               RT    11                   20   \n",
       "524550     0000000        All               RT    13                   20   \n",
       "524551     0000000        All               RT    13                   20   \n",
       "\n",
       "        students_with_scores_met_above  pct_std_met_above  \\\n",
       "0                               176261             0.4296   \n",
       "1                                83683             0.4004   \n",
       "2                                92537             0.4599   \n",
       "3                                17828             0.7296   \n",
       "4                                  557             0.1760   \n",
       "...                                ...                ...   \n",
       "524547                               0             0.0000   \n",
       "524548                              10             0.5000   \n",
       "524549                               6             0.3000   \n",
       "524550                               6             0.3000   \n",
       "524551                              10             0.5000   \n",
       "\n",
       "        pct_std_met_above_non_group  pct_std_exceeded  pct_std_met  \\\n",
       "0                               NaN            0.2340       0.1956   \n",
       "1                           0.45992            0.2127       0.1877   \n",
       "2                           0.40044            0.2560       0.2039   \n",
       "3                           0.41060            0.4301       0.2994   \n",
       "4                           0.43157            0.0730       0.1030   \n",
       "...                             ...               ...          ...   \n",
       "524547                      0.25517            0.0000       0.0000   \n",
       "524548                      0.51724            0.3000       0.2000   \n",
       "524549                      0.21831            0.1000       0.2000   \n",
       "524550                      0.21831            0.1000       0.2000   \n",
       "524551                      0.51724            0.3000       0.2000   \n",
       "\n",
       "        pct_std_met_and_above  pct_std_nearly_met  pct_std_not_met  \\\n",
       "0                      0.4296              0.2240           0.3464   \n",
       "1                      0.4004              0.2245           0.3751   \n",
       "2                      0.4599              0.2234           0.3167   \n",
       "3                      0.7296              0.1926           0.0778   \n",
       "4                      0.1760              0.2104           0.6136   \n",
       "...                       ...                 ...              ...   \n",
       "524547                 0.0000              0.1176           0.8824   \n",
       "524548                 0.5000              0.2500           0.2500   \n",
       "524549                 0.3000              0.3500           0.3500   \n",
       "524550                 0.3000              0.3500           0.3500   \n",
       "524551                 0.5000              0.2500           0.2500   \n",
       "\n",
       "        area_1_pct_above_std  area_1_pct_near_std  area_1_pct_below_std  \\\n",
       "0                     0.1747               0.5769                0.2484   \n",
       "1                     0.1588               0.5723                0.2689   \n",
       "2                     0.1912               0.5817                0.2271   \n",
       "3                     0.2983               0.6391                0.0626   \n",
       "4                     0.0581               0.4834                0.4585   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.1176                0.8824   \n",
       "524548                0.4500               0.3000                0.2500   \n",
       "524549                0.1500               0.4000                0.4500   \n",
       "524550                0.1500               0.4000                0.4500   \n",
       "524551                0.4500               0.3000                0.2500   \n",
       "\n",
       "        area_2_pct_above_std  area_2_pct_near_std  area_2_pct_below_std  \\\n",
       "0                     0.1618               0.5024                0.3358   \n",
       "1                     0.1396               0.4935                0.3669   \n",
       "2                     0.1848               0.5116                0.3036   \n",
       "3                     0.2977               0.6020                0.1002   \n",
       "4                     0.0484               0.3761                0.5755   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.1765                0.8235   \n",
       "524548                0.3500               0.3000                0.3500   \n",
       "524549                0.2500               0.3500                0.4000   \n",
       "524550                0.2500               0.3500                0.4000   \n",
       "524551                0.3500               0.3000                0.3500   \n",
       "\n",
       "        area_3_pct_above_std  area_3_pct_near_std  area_3_pct_below_std  \\\n",
       "0                     0.1175               0.7096                0.1729   \n",
       "1                     0.1147               0.7055                0.1799   \n",
       "2                     0.1204               0.7139                0.1657   \n",
       "3                     0.1818               0.7483                0.0698   \n",
       "4                     0.0496               0.6591                0.2913   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.2353                0.7647   \n",
       "524548                0.2500               0.6500                0.1000   \n",
       "524549                0.2500               0.3500                0.4000   \n",
       "524550                0.2500               0.3500                0.4000   \n",
       "524551                0.2500               0.6500                0.1000   \n",
       "\n",
       "        area_4_pct_above_std  area_4_pct_near_std  area_4_pct_below_std  \n",
       "0                     0.1607               0.6000                0.2394  \n",
       "1                     0.1520               0.5933                0.2547  \n",
       "2                     0.1696               0.6069                0.2235  \n",
       "3                     0.2887               0.6416                0.0697  \n",
       "4                     0.0632               0.5153                0.4215  \n",
       "...                      ...                  ...                   ...  \n",
       "524547                0.0000               0.0000                0.0000  \n",
       "524548                0.3000               0.3500                0.3500  \n",
       "524549                0.0000               0.0000                0.0000  \n",
       "524550                0.0000               0.0000                0.0000  \n",
       "524551                0.3000               0.3500                0.3500  \n",
       "\n",
       "[524552 rows x 30 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rename Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>n_count</th>\n",
       "      <th>met</th>\n",
       "      <th>metric_val</th>\n",
       "      <th>metric_val_nsg</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_and_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>3</td>\n",
       "      <td>410290</td>\n",
       "      <td>176261</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2340</td>\n",
       "      <td>0.1956</td>\n",
       "      <td>0.4296</td>\n",
       "      <td>0.2240</td>\n",
       "      <td>0.3464</td>\n",
       "      <td>0.1747</td>\n",
       "      <td>0.5769</td>\n",
       "      <td>0.2484</td>\n",
       "      <td>0.1618</td>\n",
       "      <td>0.5024</td>\n",
       "      <td>0.3358</td>\n",
       "      <td>0.1175</td>\n",
       "      <td>0.7096</td>\n",
       "      <td>0.1729</td>\n",
       "      <td>0.1607</td>\n",
       "      <td>0.6000</td>\n",
       "      <td>0.2394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>3</td>\n",
       "      <td>208998</td>\n",
       "      <td>83683</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.45992</td>\n",
       "      <td>0.2127</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.4004</td>\n",
       "      <td>0.2245</td>\n",
       "      <td>0.3751</td>\n",
       "      <td>0.1588</td>\n",
       "      <td>0.5723</td>\n",
       "      <td>0.2689</td>\n",
       "      <td>0.1396</td>\n",
       "      <td>0.4935</td>\n",
       "      <td>0.3669</td>\n",
       "      <td>0.1147</td>\n",
       "      <td>0.7055</td>\n",
       "      <td>0.1799</td>\n",
       "      <td>0.1520</td>\n",
       "      <td>0.5933</td>\n",
       "      <td>0.2547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>3</td>\n",
       "      <td>201212</td>\n",
       "      <td>92537</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.40044</td>\n",
       "      <td>0.2560</td>\n",
       "      <td>0.2039</td>\n",
       "      <td>0.4599</td>\n",
       "      <td>0.2234</td>\n",
       "      <td>0.3167</td>\n",
       "      <td>0.1912</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>0.2271</td>\n",
       "      <td>0.1848</td>\n",
       "      <td>0.5116</td>\n",
       "      <td>0.3036</td>\n",
       "      <td>0.1204</td>\n",
       "      <td>0.7139</td>\n",
       "      <td>0.1657</td>\n",
       "      <td>0.1696</td>\n",
       "      <td>0.6069</td>\n",
       "      <td>0.2235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RFEP</td>\n",
       "      <td>3</td>\n",
       "      <td>24435</td>\n",
       "      <td>17828</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.41060</td>\n",
       "      <td>0.4301</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.1926</td>\n",
       "      <td>0.0778</td>\n",
       "      <td>0.2983</td>\n",
       "      <td>0.6391</td>\n",
       "      <td>0.0626</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.6020</td>\n",
       "      <td>0.1002</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0.7483</td>\n",
       "      <td>0.0698</td>\n",
       "      <td>0.2887</td>\n",
       "      <td>0.6416</td>\n",
       "      <td>0.0697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023</td>\n",
       "      <td>T</td>\n",
       "      <td>415</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SM</td>\n",
       "      <td>3</td>\n",
       "      <td>3165</td>\n",
       "      <td>557</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.43157</td>\n",
       "      <td>0.0730</td>\n",
       "      <td>0.1030</td>\n",
       "      <td>0.1760</td>\n",
       "      <td>0.2104</td>\n",
       "      <td>0.6136</td>\n",
       "      <td>0.0581</td>\n",
       "      <td>0.4834</td>\n",
       "      <td>0.4585</td>\n",
       "      <td>0.0484</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.5755</td>\n",
       "      <td>0.0496</td>\n",
       "      <td>0.6591</td>\n",
       "      <td>0.2913</td>\n",
       "      <td>0.0632</td>\n",
       "      <td>0.5153</td>\n",
       "      <td>0.4215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SD</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.25517</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1765</td>\n",
       "      <td>0.8235</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2353</td>\n",
       "      <td>0.7647</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>415</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.51724</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.21831</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.21831</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.1500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>415</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.51724</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level metric_id county_code district_code  \\\n",
       "0               2023               T       415          00         00000   \n",
       "1               2023               T       415          00         00000   \n",
       "2               2023               T       415          00         00000   \n",
       "3               2023               T       415          00         00000   \n",
       "4               2023               T       415          00         00000   \n",
       "...              ...             ...       ...         ...           ...   \n",
       "524547          2018               D       416          58         72769   \n",
       "524548          2018               D       415          58         72769   \n",
       "524549          2018               D       416          58         72769   \n",
       "524550          2018               D       416          58         72769   \n",
       "524551          2018               D       415          58         72769   \n",
       "\n",
       "       school_code charter_yn reporting_category grade n_count     met  \\\n",
       "0          0000000        All                 TA     3  410290  176261   \n",
       "1          0000000        All                 GM     3  208998   83683   \n",
       "2          0000000        All                 GF     3  201212   92537   \n",
       "3          0000000        All               RFEP     3   24435   17828   \n",
       "4          0000000        All                 SM     3    3165     557   \n",
       "...            ...        ...                ...   ...     ...     ...   \n",
       "524547     0000000        All                 SD    13      17       0   \n",
       "524548     0000000        All                 RT    11      20      10   \n",
       "524549     0000000        All                 RT    11      20       6   \n",
       "524550     0000000        All                 RT    13      20       6   \n",
       "524551     0000000        All                 RT    13      20      10   \n",
       "\n",
       "        metric_val  metric_val_nsg  pct_std_exceeded  pct_std_met  \\\n",
       "0           0.4296             NaN            0.2340       0.1956   \n",
       "1           0.4004         0.45992            0.2127       0.1877   \n",
       "2           0.4599         0.40044            0.2560       0.2039   \n",
       "3           0.7296         0.41060            0.4301       0.2994   \n",
       "4           0.1760         0.43157            0.0730       0.1030   \n",
       "...            ...             ...               ...          ...   \n",
       "524547      0.0000         0.25517            0.0000       0.0000   \n",
       "524548      0.5000         0.51724            0.3000       0.2000   \n",
       "524549      0.3000         0.21831            0.1000       0.2000   \n",
       "524550      0.3000         0.21831            0.1000       0.2000   \n",
       "524551      0.5000         0.51724            0.3000       0.2000   \n",
       "\n",
       "        pct_std_met_and_above  pct_std_nearly_met  pct_std_not_met  \\\n",
       "0                      0.4296              0.2240           0.3464   \n",
       "1                      0.4004              0.2245           0.3751   \n",
       "2                      0.4599              0.2234           0.3167   \n",
       "3                      0.7296              0.1926           0.0778   \n",
       "4                      0.1760              0.2104           0.6136   \n",
       "...                       ...                 ...              ...   \n",
       "524547                 0.0000              0.1176           0.8824   \n",
       "524548                 0.5000              0.2500           0.2500   \n",
       "524549                 0.3000              0.3500           0.3500   \n",
       "524550                 0.3000              0.3500           0.3500   \n",
       "524551                 0.5000              0.2500           0.2500   \n",
       "\n",
       "        area_1_pct_above_std  area_1_pct_near_std  area_1_pct_below_std  \\\n",
       "0                     0.1747               0.5769                0.2484   \n",
       "1                     0.1588               0.5723                0.2689   \n",
       "2                     0.1912               0.5817                0.2271   \n",
       "3                     0.2983               0.6391                0.0626   \n",
       "4                     0.0581               0.4834                0.4585   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.1176                0.8824   \n",
       "524548                0.4500               0.3000                0.2500   \n",
       "524549                0.1500               0.4000                0.4500   \n",
       "524550                0.1500               0.4000                0.4500   \n",
       "524551                0.4500               0.3000                0.2500   \n",
       "\n",
       "        area_2_pct_above_std  area_2_pct_near_std  area_2_pct_below_std  \\\n",
       "0                     0.1618               0.5024                0.3358   \n",
       "1                     0.1396               0.4935                0.3669   \n",
       "2                     0.1848               0.5116                0.3036   \n",
       "3                     0.2977               0.6020                0.1002   \n",
       "4                     0.0484               0.3761                0.5755   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.1765                0.8235   \n",
       "524548                0.3500               0.3000                0.3500   \n",
       "524549                0.2500               0.3500                0.4000   \n",
       "524550                0.2500               0.3500                0.4000   \n",
       "524551                0.3500               0.3000                0.3500   \n",
       "\n",
       "        area_3_pct_above_std  area_3_pct_near_std  area_3_pct_below_std  \\\n",
       "0                     0.1175               0.7096                0.1729   \n",
       "1                     0.1147               0.7055                0.1799   \n",
       "2                     0.1204               0.7139                0.1657   \n",
       "3                     0.1818               0.7483                0.0698   \n",
       "4                     0.0496               0.6591                0.2913   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                0.0000               0.2353                0.7647   \n",
       "524548                0.2500               0.6500                0.1000   \n",
       "524549                0.2500               0.3500                0.4000   \n",
       "524550                0.2500               0.3500                0.4000   \n",
       "524551                0.2500               0.6500                0.1000   \n",
       "\n",
       "        area_4_pct_above_std  area_4_pct_near_std  area_4_pct_below_std  \n",
       "0                     0.1607               0.6000                0.2394  \n",
       "1                     0.1520               0.5933                0.2547  \n",
       "2                     0.1696               0.6069                0.2235  \n",
       "3                     0.2887               0.6416                0.0697  \n",
       "4                     0.0632               0.5153                0.4215  \n",
       "...                      ...                  ...                   ...  \n",
       "524547                0.0000               0.0000                0.0000  \n",
       "524548                0.3000               0.3500                0.3500  \n",
       "524549                0.0000               0.0000                0.0000  \n",
       "524550                0.0000               0.0000                0.0000  \n",
       "524551                0.3000               0.3500                0.3500  \n",
       "\n",
       "[524552 rows x 30 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df = caaspp_df.rename(columns={\n",
    "    'test_year': 'academic_year',\n",
    "    'test_id': 'metric_id',\n",
    "    'student_group_id': 'reporting_category',\n",
    "    'students_with_scores': 'n_count',\n",
    "    'students_with_scores_met_above': 'met',\n",
    "    'pct_std_met_above': 'metric_val',\n",
    "    'pct_std_met_above_non_group': 'metric_val_nsg'\n",
    "})\n",
    "caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "caaspp_df['grade'] = caaspp_df['grade'].apply(lambda x: 'All Grades' if x == '13' else 'Grade ' + x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Grade 3', 'Grade 4', 'Grade 5', 'Grade 6', 'Grade 7', 'Grade 8',\n",
       "       'Grade 11', 'All Grades'], dtype=object)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caaspp_df.grade.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>n_count</th>\n",
       "      <th>met</th>\n",
       "      <th>metric_val</th>\n",
       "      <th>metric_val_nsg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2233</td>\n",
       "      <td>38</td>\n",
       "      <td>0.01702</td>\n",
       "      <td>0.02636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2419</td>\n",
       "      <td>64</td>\n",
       "      <td>0.02646</td>\n",
       "      <td>0.01695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690</td>\n",
       "      <td>2</td>\n",
       "      <td>0.00290</td>\n",
       "      <td>0.02518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>859</td>\n",
       "      <td>53</td>\n",
       "      <td>0.06170</td>\n",
       "      <td>0.01289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>601</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>99</td>\n",
       "      <td>2</td>\n",
       "      <td>0.02020</td>\n",
       "      <td>0.02192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183303</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3646836</td>\n",
       "      <td>169199</td>\n",
       "      <td>0.04640</td>\n",
       "      <td>0.01974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183304</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>4037882</td>\n",
       "      <td>179640</td>\n",
       "      <td>0.04449</td>\n",
       "      <td>0.01884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183305</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6384919</td>\n",
       "      <td>223867</td>\n",
       "      <td>0.03506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183306</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5748545</td>\n",
       "      <td>210691</td>\n",
       "      <td>0.03665</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183307</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>601</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>753611</td>\n",
       "      <td>13712</td>\n",
       "      <td>0.01820</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>183308 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               D        601          01         10017   \n",
       "1            2022-23               D        601          01         10017   \n",
       "2            2022-23               D        601          01         10017   \n",
       "3            2022-23               D        601          01         10017   \n",
       "4            2022-23               D        601          01         10017   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "183303       2017-18               T        601          00         00000   \n",
       "183304       2017-18               T        601          00         00000   \n",
       "183305       2017-18               T        601          00         00000   \n",
       "183306       2017-18               T        601          00         00000   \n",
       "183307       2017-18               T        601          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  n_count     met  \\\n",
       "0          0000000        All                 GF  All Grades     2233      38   \n",
       "1          0000000        All                 GM  All Grades     2419      64   \n",
       "2          0000000        All                 RA  All Grades      690       2   \n",
       "3          0000000        All                 RB  All Grades      859      53   \n",
       "4          0000000        All                 RD  All Grades       99       2   \n",
       "...            ...        ...                ...         ...      ...     ...   \n",
       "183303     0000000        No                  SS  All Grades  3646836  169199   \n",
       "183304     0000000        All                 SS  All Grades  4037882  179640   \n",
       "183305     0000000        All                 TA  All Grades  6384919  223867   \n",
       "183306     0000000        No                  TA  All Grades  5748545  210691   \n",
       "183307     0000000        Yes                 TA  All Grades   753611   13712   \n",
       "\n",
       "        metric_val  metric_val_nsg  \n",
       "0          0.01702         0.02636  \n",
       "1          0.02646         0.01695  \n",
       "2          0.00290         0.02518  \n",
       "3          0.06170         0.01289  \n",
       "4          0.02020         0.02192  \n",
       "...            ...             ...  \n",
       "183303     0.04640         0.01974  \n",
       "183304     0.04449         0.01884  \n",
       "183305     0.03506             NaN  \n",
       "183306     0.03665             NaN  \n",
       "183307     0.01820             NaN  \n",
       "\n",
       "[183308 rows x 13 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suspension_df = suspension_df.rename(columns={\n",
    "    'cumulative_enrollment': 'n_count',\n",
    "    'unduplicated_students_suspended_total': 'met',\n",
    "    'suspension_rate_total': 'metric_val',\n",
    "    'suspension_rate_total_non_group': 'metric_val_nsg'\n",
    "})\n",
    "\n",
    "suspension_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2022-23', '2021-22', '2020-21', '2019-20', '2018-19', '2017-18'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suspension_df.academic_year.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>n_count</th>\n",
       "      <th>met</th>\n",
       "      <th>metric_val</th>\n",
       "      <th>metric_val_nsg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2233</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2419</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>859</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>D</td>\n",
       "      <td>602</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RD</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>99</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191359</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>4037882</td>\n",
       "      <td>4539</td>\n",
       "      <td>0.00112</td>\n",
       "      <td>0.00034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191360</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>478701</td>\n",
       "      <td>166</td>\n",
       "      <td>0.00035</td>\n",
       "      <td>0.00008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191361</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6384919</td>\n",
       "      <td>5326</td>\n",
       "      <td>0.00083</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191362</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>753611</td>\n",
       "      <td>189</td>\n",
       "      <td>0.00025</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191363</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>602</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5748545</td>\n",
       "      <td>5138</td>\n",
       "      <td>0.00089</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191364 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               D        602          01         10017   \n",
       "1            2022-23               D        602          01         10017   \n",
       "2            2022-23               D        602          01         10017   \n",
       "3            2022-23               D        602          01         10017   \n",
       "4            2022-23               D        602          01         10017   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "191359       2017-18               T        602          00         00000   \n",
       "191360       2017-18               T        602          00         00000   \n",
       "191361       2017-18               T        602          00         00000   \n",
       "191362       2017-18               T        602          00         00000   \n",
       "191363       2017-18               T        602          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  n_count   met  \\\n",
       "0          0000000        All                 GF  All Grades     2233     0   \n",
       "1          0000000        All                 GM  All Grades     2419     0   \n",
       "2          0000000        All                 RA  All Grades      690     0   \n",
       "3          0000000        All                 RB  All Grades      859     0   \n",
       "4          0000000        All                 RD  All Grades       99     0   \n",
       "...            ...        ...                ...         ...      ...   ...   \n",
       "191359     0000000        All                 SS  All Grades  4037882  4539   \n",
       "191360     0000000        Yes                 SS  All Grades   478701   166   \n",
       "191361     0000000        All                 TA  All Grades  6384919  5326   \n",
       "191362     0000000        Yes                 TA  All Grades   753611   189   \n",
       "191363     0000000        No                  TA  All Grades  5748545  5138   \n",
       "\n",
       "        metric_val  metric_val_nsg  \n",
       "0          0.00000         0.00000  \n",
       "1          0.00000         0.00000  \n",
       "2          0.00000         0.00000  \n",
       "3          0.00000         0.00000  \n",
       "4          0.00000         0.00000  \n",
       "...            ...             ...  \n",
       "191359     0.00112         0.00034  \n",
       "191360     0.00035         0.00008  \n",
       "191361     0.00083             NaN  \n",
       "191362     0.00025             NaN  \n",
       "191363     0.00089             NaN  \n",
       "\n",
       "[191364 rows x 13 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expulsion_df = expulsion_df.rename(columns={\n",
    "    'cumulative_enrollment': 'n_count',\n",
    "    'expelled_students_total': 'met',\n",
    "    'expulsion_rate_total': 'metric_val',\n",
    "    'expulsion_rate_total_non_group': 'metric_val_nsg'\n",
    "})\n",
    "\n",
    "expulsion_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>n_count</th>\n",
       "      <th>met</th>\n",
       "      <th>metric_val</th>\n",
       "      <th>metric_val_nsg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2893518</td>\n",
       "      <td>719562</td>\n",
       "      <td>0.24868</td>\n",
       "      <td>0.25013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3059625</td>\n",
       "      <td>764487</td>\n",
       "      <td>0.24986</td>\n",
       "      <td>0.24896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GX</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5746</td>\n",
       "      <td>2253</td>\n",
       "      <td>0.39210</td>\n",
       "      <td>0.24929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>578202</td>\n",
       "      <td>58964</td>\n",
       "      <td>0.10198</td>\n",
       "      <td>0.26527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>301921</td>\n",
       "      <td>110537</td>\n",
       "      <td>0.36611</td>\n",
       "      <td>0.24320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150450</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3610723</td>\n",
       "      <td>499336</td>\n",
       "      <td>0.13829</td>\n",
       "      <td>0.06275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150451</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SS</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>438944</td>\n",
       "      <td>73929</td>\n",
       "      <td>0.16842</td>\n",
       "      <td>0.07512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150452</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>6315131</td>\n",
       "      <td>702531</td>\n",
       "      <td>0.11125</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150453</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>No</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5688600</td>\n",
       "      <td>629719</td>\n",
       "      <td>0.11070</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150454</th>\n",
       "      <td>2017-18</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>Yes</td>\n",
       "      <td>TA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>690446</td>\n",
       "      <td>92822</td>\n",
       "      <td>0.13444</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150455 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level  metric_id county_code district_code  \\\n",
       "0            2022-23               T        502          00         00000   \n",
       "1            2022-23               T        502          00         00000   \n",
       "2            2022-23               T        502          00         00000   \n",
       "3            2022-23               T        502          00         00000   \n",
       "4            2022-23               T        502          00         00000   \n",
       "...              ...             ...        ...         ...           ...   \n",
       "150450       2017-18               T        502          00         00000   \n",
       "150451       2017-18               T        502          00         00000   \n",
       "150452       2017-18               T        502          00         00000   \n",
       "150453       2017-18               T        502          00         00000   \n",
       "150454       2017-18               T        502          00         00000   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  n_count     met  \\\n",
       "0          0000000        All                 GF  All Grades  2893518  719562   \n",
       "1          0000000        All                 GM  All Grades  3059625  764487   \n",
       "2          0000000        All                 GX  All Grades     5746    2253   \n",
       "3          0000000        All                 RA  All Grades   578202   58964   \n",
       "4          0000000        All                 RB  All Grades   301921  110537   \n",
       "...            ...        ...                ...         ...      ...     ...   \n",
       "150450     0000000         No                 SS  All Grades  3610723  499336   \n",
       "150451     0000000        Yes                 SS  All Grades   438944   73929   \n",
       "150452     0000000        All                 TA  All Grades  6315131  702531   \n",
       "150453     0000000         No                 TA  All Grades  5688600  629719   \n",
       "150454     0000000        Yes                 TA  All Grades   690446   92822   \n",
       "\n",
       "        metric_val  metric_val_nsg  \n",
       "0          0.24868         0.25013  \n",
       "1          0.24986         0.24896  \n",
       "2          0.39210         0.24929  \n",
       "3          0.10198         0.26527  \n",
       "4          0.36611         0.24320  \n",
       "...            ...             ...  \n",
       "150450     0.13829         0.06275  \n",
       "150451     0.16842         0.07512  \n",
       "150452     0.11125             NaN  \n",
       "150453     0.11070             NaN  \n",
       "150454     0.13444             NaN  \n",
       "\n",
       "[150455 rows x 13 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "absenteeism_df = absenteeism_df.rename(columns={\n",
    "    'enrollment_count': 'n_count',\n",
    "    'absenteeism_count': 'met',\n",
    "    'absenteeism_rate': 'metric_val',\n",
    "    'absenteeism_rate_non_group': 'metric_val_nsg'\n",
    "})\n",
    "\n",
    "absenteeism_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>academic_year</th>\n",
       "      <th>aggregate_level</th>\n",
       "      <th>metric_id</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>charter_yn</th>\n",
       "      <th>reporting_category</th>\n",
       "      <th>grade</th>\n",
       "      <th>n_count</th>\n",
       "      <th>met</th>\n",
       "      <th>metric_val</th>\n",
       "      <th>metric_val_nsg</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_and_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GF</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>2893518</td>\n",
       "      <td>719562</td>\n",
       "      <td>0.24868</td>\n",
       "      <td>0.25013</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GM</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>3059625</td>\n",
       "      <td>764487</td>\n",
       "      <td>0.24986</td>\n",
       "      <td>0.24896</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>GX</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>5746</td>\n",
       "      <td>2253</td>\n",
       "      <td>0.39210</td>\n",
       "      <td>0.24929</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RA</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>578202</td>\n",
       "      <td>58964</td>\n",
       "      <td>0.10198</td>\n",
       "      <td>0.26527</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-23</td>\n",
       "      <td>T</td>\n",
       "      <td>502</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RB</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>301921</td>\n",
       "      <td>110537</td>\n",
       "      <td>0.36611</td>\n",
       "      <td>0.24320</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524547</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>SD</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.25517</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.8824</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.1765</td>\n",
       "      <td>0.8235</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.2353</td>\n",
       "      <td>0.7647</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524548</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>415</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>Grade 11</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.50000</td>\n",
       "      <td>0.51724</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524549</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>Grade 11</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.30000</td>\n",
       "      <td>0.21831</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524550</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>416</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>0.30000</td>\n",
       "      <td>0.21831</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.4500</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524551</th>\n",
       "      <td>2018</td>\n",
       "      <td>D</td>\n",
       "      <td>415</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>All</td>\n",
       "      <td>RT</td>\n",
       "      <td>All Grades</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.50000</td>\n",
       "      <td>0.51724</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3500</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.35</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1049679 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       academic_year aggregate_level metric_id county_code district_code  \\\n",
       "0            2022-23               T       502          00         00000   \n",
       "1            2022-23               T       502          00         00000   \n",
       "2            2022-23               T       502          00         00000   \n",
       "3            2022-23               T       502          00         00000   \n",
       "4            2022-23               T       502          00         00000   \n",
       "...              ...             ...       ...         ...           ...   \n",
       "524547          2018               D       416          58         72769   \n",
       "524548          2018               D       415          58         72769   \n",
       "524549          2018               D       416          58         72769   \n",
       "524550          2018               D       416          58         72769   \n",
       "524551          2018               D       415          58         72769   \n",
       "\n",
       "       school_code charter_yn reporting_category       grade  n_count     met  \\\n",
       "0          0000000        All                 GF  All Grades  2893518  719562   \n",
       "1          0000000        All                 GM  All Grades  3059625  764487   \n",
       "2          0000000        All                 GX  All Grades     5746    2253   \n",
       "3          0000000        All                 RA  All Grades   578202   58964   \n",
       "4          0000000        All                 RB  All Grades   301921  110537   \n",
       "...            ...        ...                ...         ...      ...     ...   \n",
       "524547     0000000        All                 SD  All Grades       17       0   \n",
       "524548     0000000        All                 RT    Grade 11       20      10   \n",
       "524549     0000000        All                 RT    Grade 11       20       6   \n",
       "524550     0000000        All                 RT  All Grades       20       6   \n",
       "524551     0000000        All                 RT  All Grades       20      10   \n",
       "\n",
       "        metric_val  metric_val_nsg  pct_std_exceeded  pct_std_met  \\\n",
       "0          0.24868         0.25013               NaN          NaN   \n",
       "1          0.24986         0.24896               NaN          NaN   \n",
       "2          0.39210         0.24929               NaN          NaN   \n",
       "3          0.10198         0.26527               NaN          NaN   \n",
       "4          0.36611         0.24320               NaN          NaN   \n",
       "...            ...             ...               ...          ...   \n",
       "524547     0.00000         0.25517               0.0          0.0   \n",
       "524548     0.50000         0.51724               0.3          0.2   \n",
       "524549     0.30000         0.21831               0.1          0.2   \n",
       "524550     0.30000         0.21831               0.1          0.2   \n",
       "524551     0.50000         0.51724               0.3          0.2   \n",
       "\n",
       "        pct_std_met_and_above  pct_std_nearly_met  pct_std_not_met  \\\n",
       "0                         NaN                 NaN              NaN   \n",
       "1                         NaN                 NaN              NaN   \n",
       "2                         NaN                 NaN              NaN   \n",
       "3                         NaN                 NaN              NaN   \n",
       "4                         NaN                 NaN              NaN   \n",
       "...                       ...                 ...              ...   \n",
       "524547                    0.0              0.1176           0.8824   \n",
       "524548                    0.5              0.2500           0.2500   \n",
       "524549                    0.3              0.3500           0.3500   \n",
       "524550                    0.3              0.3500           0.3500   \n",
       "524551                    0.5              0.2500           0.2500   \n",
       "\n",
       "        area_1_pct_above_std  area_1_pct_near_std  area_1_pct_below_std  \\\n",
       "0                        NaN                  NaN                   NaN   \n",
       "1                        NaN                  NaN                   NaN   \n",
       "2                        NaN                  NaN                   NaN   \n",
       "3                        NaN                  NaN                   NaN   \n",
       "4                        NaN                  NaN                   NaN   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                  0.00               0.1176                0.8824   \n",
       "524548                  0.45               0.3000                0.2500   \n",
       "524549                  0.15               0.4000                0.4500   \n",
       "524550                  0.15               0.4000                0.4500   \n",
       "524551                  0.45               0.3000                0.2500   \n",
       "\n",
       "        area_2_pct_above_std  area_2_pct_near_std  area_2_pct_below_std  \\\n",
       "0                        NaN                  NaN                   NaN   \n",
       "1                        NaN                  NaN                   NaN   \n",
       "2                        NaN                  NaN                   NaN   \n",
       "3                        NaN                  NaN                   NaN   \n",
       "4                        NaN                  NaN                   NaN   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                  0.00               0.1765                0.8235   \n",
       "524548                  0.35               0.3000                0.3500   \n",
       "524549                  0.25               0.3500                0.4000   \n",
       "524550                  0.25               0.3500                0.4000   \n",
       "524551                  0.35               0.3000                0.3500   \n",
       "\n",
       "        area_3_pct_above_std  area_3_pct_near_std  area_3_pct_below_std  \\\n",
       "0                        NaN                  NaN                   NaN   \n",
       "1                        NaN                  NaN                   NaN   \n",
       "2                        NaN                  NaN                   NaN   \n",
       "3                        NaN                  NaN                   NaN   \n",
       "4                        NaN                  NaN                   NaN   \n",
       "...                      ...                  ...                   ...   \n",
       "524547                  0.00               0.2353                0.7647   \n",
       "524548                  0.25               0.6500                0.1000   \n",
       "524549                  0.25               0.3500                0.4000   \n",
       "524550                  0.25               0.3500                0.4000   \n",
       "524551                  0.25               0.6500                0.1000   \n",
       "\n",
       "        area_4_pct_above_std  area_4_pct_near_std  area_4_pct_below_std  \n",
       "0                        NaN                  NaN                   NaN  \n",
       "1                        NaN                  NaN                   NaN  \n",
       "2                        NaN                  NaN                   NaN  \n",
       "3                        NaN                  NaN                   NaN  \n",
       "4                        NaN                  NaN                   NaN  \n",
       "...                      ...                  ...                   ...  \n",
       "524547                   0.0                 0.00                  0.00  \n",
       "524548                   0.3                 0.35                  0.35  \n",
       "524549                   0.0                 0.00                  0.00  \n",
       "524550                   0.0                 0.00                  0.00  \n",
       "524551                   0.3                 0.35                  0.35  \n",
       "\n",
       "[1049679 rows x 30 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset_df = pd.concat([absenteeism_df, \n",
    "                          expulsion_df, \n",
    "                          suspension_df, \n",
    "                          caaspp_df])\n",
    "final_dataset_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the mapping from metric_id\n",
    "year_mapping = {\n",
    "    '2017-18': '2018',\n",
    "    '2018-19': '2019',\n",
    "    '2019-20': '2020',\n",
    "    '2020-21': '2021',\n",
    "    '2021-22': '2022',\n",
    "    '2022-23': '2023'\n",
    "}\n",
    "\n",
    "# Use .loc to ensure you're modifying the original DataFrame\n",
    "final_dataset_df.loc[:, 'academic_year'] = final_dataset_df['academic_year'].astype(str).replace(year_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of columns you want to ensure are strings\n",
    "columns_to_convert = ['county_code', 'district_code', 'school_code']\n",
    "\n",
    "# Convert these columns to strings in the DataFrame\n",
    "for column in columns_to_convert:\n",
    "    final_dataset_df[column] = final_dataset_df[column].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset_df['charter_yn'] = final_dataset_df['charter_yn'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset_df['aggregate_level'] = final_dataset_df['aggregate_level'].replace({'T': 'State', 'D': 'District', 'S': 'School'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset_df['charter_yn'] = final_dataset_df['charter_yn'].replace({'No': 'Non-Charter Schools', 'Yes': 'Charter Schools'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "condation1_is_district_row = (final_dataset_df['aggregate_level'] == 'District')\n",
    "condition2_has_charter_school = (final_dataset_df['charter_yn'] == 'Charter Schools')\n",
    "\n",
    "comb_conditions = (condation1_is_district_row & condition2_has_charter_school)\n",
    "\n",
    "filtered_district_df = final_dataset_df[comb_conditions]\n",
    "\n",
    "# Step 2: Select distinct rows based on 'school_code' and 'type_id'\n",
    "final_filtered_district_df = filtered_district_df[['academic_year', 'aggregate_level', 'metric_id', 'county_code', 'district_code', 'reporting_category', 'grade', 'charter_yn']].drop_duplicates()\n",
    "final_filtered_district_df = final_filtered_district_df.rename(columns={'charter_yn': 'check_charter'})\n",
    "\n",
    "final_dataset_df = pd.merge(final_dataset_df, final_filtered_district_df, on = ['academic_year','aggregate_level', 'metric_id', 'county_code', 'district_code', 'reporting_category', 'grade'], how='left')\n",
    "\n",
    "# final_dataset_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "condation1_is_district_row = (final_dataset_df['aggregate_level'] == 'District')\n",
    "condition2_has_all = (final_dataset_df['charter_yn'] == 'All')\n",
    "\n",
    "comb_conditions = (condation1_is_district_row & condition2_has_all)\n",
    "\n",
    "filtered_district_df = final_dataset_df[comb_conditions]\n",
    "\n",
    "# Step 2: Select distinct rows based on 'school_code' and 'type_id'\n",
    "final_filtered_district_all_df = filtered_district_df[['academic_year', 'aggregate_level', 'metric_id', 'county_code', 'district_code', 'reporting_category', 'grade', 'charter_yn']].drop_duplicates()\n",
    "\n",
    "final_filtered_district_all_df = final_filtered_district_all_df.rename(columns={'charter_yn': 'check_all'})\n",
    "\n",
    "final_dataset_df = pd.merge(final_dataset_df, final_filtered_district_all_df, on = ['academic_year','aggregate_level', 'metric_id', 'county_code', 'district_code', 'reporting_category', 'grade'], how='left')\n",
    "# final_dataset_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "condation1_is_district_row = (final_dataset_df['aggregate_level'] == 'District')\n",
    "condition2_non_charter_school = (final_dataset_df['charter_yn'] == 'Non-Charter Schools')\n",
    "condition3_no_charter_school = (final_dataset_df['check_charter'].isna())\n",
    "condition4_has_all = (final_dataset_df['check_all'] == 'All')\n",
    "\n",
    "full_conditions = (condation1_is_district_row & condition2_non_charter_school & condition3_no_charter_school & condition4_has_all)\n",
    "final_dataset_df = final_dataset_df[~full_conditions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset_df = final_dataset_df.drop(columns=['check_charter', 'check_all'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset_df.to_csv('tableau_ready_files/metric_values_fact.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}