create_caaspp_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>School Name</th>\n",
       "      <th>Auth (14)</th>\n",
       "      <th>Program (16)</th>\n",
       "      <th>Charter (14)</th>\n",
       "      <th>Facility (4)</th>\n",
       "      <th>School Code</th>\n",
       "      <th>School Type</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18</td>\n",
       "      <td>Greenwood Academy</td>\n",
       "      <td>WCCUSD</td>\n",
       "      <td>Non-ILCoP</td>\n",
       "      <td>WCCUSD School</td>\n",
       "      <td>No</td>\n",
       "      <td>0733253</td>\n",
       "      <td>Alternative</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63</td>\n",
       "      <td>Vista Independent Study/Virtual Academy</td>\n",
       "      <td>WCCUSD</td>\n",
       "      <td>Non-ILCoP</td>\n",
       "      <td>WCCUSD School</td>\n",
       "      <td>No</td>\n",
       "      <td>0730325</td>\n",
       "      <td>Alternative</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Aspire Richmond Technology Academy</td>\n",
       "      <td>WCCUSD</td>\n",
       "      <td>Non-ILCoP</td>\n",
       "      <td>WCC Charter</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0132118</td>\n",
       "      <td>Elementary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Bayview Elementary</td>\n",
       "      <td>WCCUSD</td>\n",
       "      <td>ILCoP</td>\n",
       "      <td>WCCUSD School</td>\n",
       "      <td>No</td>\n",
       "      <td>6004600</td>\n",
       "      <td>Elementary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>Cesar E. Chavez Elementary</td>\n",
       "      <td>WCCUSD</td>\n",
       "      <td>ILCoP</td>\n",
       "      <td>WCCUSD School</td>\n",
       "      <td>No</td>\n",
       "      <td>6114094</td>\n",
       "      <td>Elementary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>1</td>\n",
       "      <td>Aspire Ca College Preparatory Academy</td>\n",
       "      <td>WCCUSD</td>\n",
       "      <td>Non-ILCoP</td>\n",
       "      <td>WCC Charter</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0132100</td>\n",
       "      <td>Middle-High</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>25</td>\n",
       "      <td>Invictus Academy of Richmond</td>\n",
       "      <td>CCCOE</td>\n",
       "      <td>Non-ILCoP</td>\n",
       "      <td>WCC Charter</td>\n",
       "      <td>No</td>\n",
       "      <td>0137026</td>\n",
       "      <td>Middle-High</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>35</td>\n",
       "      <td>Making Waves Academy</td>\n",
       "      <td>CCCOE</td>\n",
       "      <td>Non-ILCoP</td>\n",
       "      <td>WCC Charter</td>\n",
       "      <td>No</td>\n",
       "      <td>0114470</td>\n",
       "      <td>Middle-High</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>58</td>\n",
       "      <td>Summit: K2</td>\n",
       "      <td>CCCOE</td>\n",
       "      <td>Non-ILCoP</td>\n",
       "      <td>WCC Charter</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0129684</td>\n",
       "      <td>Middle-High</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>59</td>\n",
       "      <td>Summit: Tamalpais</td>\n",
       "      <td>WCCUSD</td>\n",
       "      <td>Non-ILCoP</td>\n",
       "      <td>WCC Charter</td>\n",
       "      <td>No</td>\n",
       "      <td>0133637</td>\n",
       "      <td>Middle-High</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>66 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     #                              School Name Auth (14) Program (16)  \\\n",
       "0   18                        Greenwood Academy    WCCUSD    Non-ILCoP   \n",
       "1   63  Vista Independent Study/Virtual Academy    WCCUSD    Non-ILCoP   \n",
       "2    2       Aspire Richmond Technology Academy    WCCUSD    Non-ILCoP   \n",
       "3    3                       Bayview Elementary    WCCUSD        ILCoP   \n",
       "4    6               Cesar E. Chavez Elementary    WCCUSD        ILCoP   \n",
       "..  ..                                      ...       ...          ...   \n",
       "61   1    Aspire Ca College Preparatory Academy    WCCUSD    Non-ILCoP   \n",
       "62  25             Invictus Academy of Richmond     CCCOE    Non-ILCoP   \n",
       "63  35                     Making Waves Academy     CCCOE    Non-ILCoP   \n",
       "64  58                               Summit: K2     CCCOE    Non-ILCoP   \n",
       "65  59                        Summit: Tamalpais    WCCUSD    Non-ILCoP   \n",
       "\n",
       "    Charter (14)  Facility (4) School Code  School Type Note  \n",
       "0   WCCUSD School           No     0733253  Alternative  NaN  \n",
       "1   WCCUSD School           No     0730325  Alternative  NaN  \n",
       "2     WCC Charter          Yes     0132118   Elementary  NaN  \n",
       "3   WCCUSD School           No     6004600   Elementary  NaN  \n",
       "4   WCCUSD School           No     6114094   Elementary  NaN  \n",
       "..            ...          ...         ...          ...  ...  \n",
       "61    WCC Charter          Yes     0132100  Middle-High  NaN  \n",
       "62    WCC Charter           No     0137026  Middle-High  NaN  \n",
       "63    WCC Charter           No     0114470  Middle-High  NaN  \n",
       "64    WCC Charter          Yes     0129684  Middle-High  NaN  \n",
       "65    WCC Charter           No     0133637  Middle-High  NaN  \n",
       "\n",
       "[66 rows x 9 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wcc_schools_df = pd.read_csv('cef_provided_files/cef_school_list.csv', encoding='ISO-8859-1',dtype=str)\n",
    "wcc_schools_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2023"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2960990</td>\n",
       "      <td>2957491</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>420238</td>\n",
       "      <td>410559</td>\n",
       "      <td>2409.8</td>\n",
       "      <td>23.40</td>\n",
       "      <td>19.56</td>\n",
       "      <td>42.96</td>\n",
       "      <td>22.40</td>\n",
       "      <td>34.64</td>\n",
       "      <td>410290</td>\n",
       "      <td>17.47</td>\n",
       "      <td>57.69</td>\n",
       "      <td>24.84</td>\n",
       "      <td>16.18</td>\n",
       "      <td>50.24</td>\n",
       "      <td>33.58</td>\n",
       "      <td>11.75</td>\n",
       "      <td>70.96</td>\n",
       "      <td>17.29</td>\n",
       "      <td>16.07</td>\n",
       "      <td>60.00</td>\n",
       "      <td>23.94</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1512977</td>\n",
       "      <td>1511175</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>214436</td>\n",
       "      <td>209138</td>\n",
       "      <td>2402.4</td>\n",
       "      <td>21.27</td>\n",
       "      <td>18.77</td>\n",
       "      <td>40.04</td>\n",
       "      <td>22.45</td>\n",
       "      <td>37.51</td>\n",
       "      <td>208998</td>\n",
       "      <td>15.88</td>\n",
       "      <td>57.23</td>\n",
       "      <td>26.89</td>\n",
       "      <td>13.96</td>\n",
       "      <td>49.35</td>\n",
       "      <td>36.69</td>\n",
       "      <td>11.47</td>\n",
       "      <td>70.55</td>\n",
       "      <td>17.99</td>\n",
       "      <td>15.20</td>\n",
       "      <td>59.33</td>\n",
       "      <td>25.47</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1445927</td>\n",
       "      <td>1444241</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>205715</td>\n",
       "      <td>201341</td>\n",
       "      <td>2417.5</td>\n",
       "      <td>25.60</td>\n",
       "      <td>20.39</td>\n",
       "      <td>45.99</td>\n",
       "      <td>22.34</td>\n",
       "      <td>31.67</td>\n",
       "      <td>201212</td>\n",
       "      <td>19.12</td>\n",
       "      <td>58.17</td>\n",
       "      <td>22.71</td>\n",
       "      <td>18.48</td>\n",
       "      <td>51.16</td>\n",
       "      <td>30.36</td>\n",
       "      <td>12.04</td>\n",
       "      <td>71.39</td>\n",
       "      <td>16.57</td>\n",
       "      <td>16.96</td>\n",
       "      <td>60.69</td>\n",
       "      <td>22.35</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>6</td>\n",
       "      <td>B</td>\n",
       "      <td>2459254</td>\n",
       "      <td>2456394</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>321484</td>\n",
       "      <td>316733</td>\n",
       "      <td>2428.5</td>\n",
       "      <td>28.93</td>\n",
       "      <td>21.99</td>\n",
       "      <td>50.92</td>\n",
       "      <td>21.81</td>\n",
       "      <td>27.27</td>\n",
       "      <td>316516</td>\n",
       "      <td>21.56</td>\n",
       "      <td>59.05</td>\n",
       "      <td>19.39</td>\n",
       "      <td>19.88</td>\n",
       "      <td>52.83</td>\n",
       "      <td>27.29</td>\n",
       "      <td>14.03</td>\n",
       "      <td>71.81</td>\n",
       "      <td>14.16</td>\n",
       "      <td>19.54</td>\n",
       "      <td>61.39</td>\n",
       "      <td>19.08</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>7</td>\n",
       "      <td>B</td>\n",
       "      <td>144219</td>\n",
       "      <td>144145</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>27927</td>\n",
       "      <td>27764</td>\n",
       "      <td>2462.9</td>\n",
       "      <td>40.62</td>\n",
       "      <td>23.73</td>\n",
       "      <td>64.35</td>\n",
       "      <td>19.87</td>\n",
       "      <td>15.78</td>\n",
       "      <td>27760</td>\n",
       "      <td>30.24</td>\n",
       "      <td>58.31</td>\n",
       "      <td>11.45</td>\n",
       "      <td>28.50</td>\n",
       "      <td>54.52</td>\n",
       "      <td>16.98</td>\n",
       "      <td>19.43</td>\n",
       "      <td>71.38</td>\n",
       "      <td>9.19</td>\n",
       "      <td>28.13</td>\n",
       "      <td>60.23</td>\n",
       "      <td>11.65</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052589</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>226</td>\n",
       "      <td>B</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.33</td>\n",
       "      <td>20.83</td>\n",
       "      <td>29.17</td>\n",
       "      <td>33.33</td>\n",
       "      <td>37.50</td>\n",
       "      <td>24</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052590</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>8</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052591</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>241</td>\n",
       "      <td>B</td>\n",
       "      <td>244</td>\n",
       "      <td>243</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>263</td>\n",
       "      <td>244</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.76</td>\n",
       "      <td>27.16</td>\n",
       "      <td>32.92</td>\n",
       "      <td>25.10</td>\n",
       "      <td>41.98</td>\n",
       "      <td>243</td>\n",
       "      <td>10.29</td>\n",
       "      <td>47.74</td>\n",
       "      <td>41.98</td>\n",
       "      <td>15.23</td>\n",
       "      <td>64.20</td>\n",
       "      <td>20.58</td>\n",
       "      <td>12.76</td>\n",
       "      <td>66.67</td>\n",
       "      <td>20.58</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052592</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>250</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>8</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052593</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>252</td>\n",
       "      <td>B</td>\n",
       "      <td>198</td>\n",
       "      <td>197</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>216</td>\n",
       "      <td>198</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.09</td>\n",
       "      <td>29.44</td>\n",
       "      <td>35.53</td>\n",
       "      <td>26.40</td>\n",
       "      <td>38.07</td>\n",
       "      <td>197</td>\n",
       "      <td>9.14</td>\n",
       "      <td>50.76</td>\n",
       "      <td>40.10</td>\n",
       "      <td>16.75</td>\n",
       "      <td>64.47</td>\n",
       "      <td>18.78</td>\n",
       "      <td>13.71</td>\n",
       "      <td>68.53</td>\n",
       "      <td>17.77</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4052594 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2023   \n",
       "1                00         00000     0000000    NaN      2023   \n",
       "2                00         00000     0000000    NaN      2023   \n",
       "3                00         00000     0000000    NaN      2023   \n",
       "4                00         00000     0000000    NaN      2023   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "4052589          58         72769     5838305    NaN      2023   \n",
       "4052590          58         72769     5838305    NaN      2023   \n",
       "4052591          58         72769     5838305    NaN      2023   \n",
       "4052592          58         72769     5838305    NaN      2023   \n",
       "4052593          58         72769     5838305    NaN      2023   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    2960990   \n",
       "1                      3         B                    1512977   \n",
       "2                      4         B                    1445927   \n",
       "3                      6         B                    2459254   \n",
       "4                      7         B                     144219   \n",
       "...                  ...       ...                        ...   \n",
       "4052589              226         B                         24   \n",
       "4052590              227         B                          8   \n",
       "4052591              241         B                        244   \n",
       "4052592              250         B                          8   \n",
       "4052593              252         B                        198   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        2957491     3       1            420238   \n",
       "1                        1511175     3       1            214436   \n",
       "2                        1444241     3       1            205715   \n",
       "3                        2456394     3       1            321484   \n",
       "4                         144145     3       1             27927   \n",
       "...                          ...   ...     ...               ...   \n",
       "4052589                       24    13       2                28   \n",
       "4052590                        8    13       2                 8   \n",
       "4052591                      243    13       2               263   \n",
       "4052592                        8    13       2                 8   \n",
       "4052593                      197    13       2               216   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                410559           2409.8            23.40       19.56   \n",
       "1                209138           2402.4            21.27       18.77   \n",
       "2                201341           2417.5            25.60       20.39   \n",
       "3                316733           2428.5            28.93       21.99   \n",
       "4                 27764           2462.9            40.62       23.73   \n",
       "...                 ...              ...              ...         ...   \n",
       "4052589              24              NaN             8.33       20.83   \n",
       "4052590               8              NaN                *           *   \n",
       "4052591             244              NaN             5.76       27.16   \n",
       "4052592               8              NaN                *           *   \n",
       "4052593             198              NaN             6.09       29.44   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   42.96              22.40           34.64   \n",
       "1                   40.04              22.45           37.51   \n",
       "2                   45.99              22.34           31.67   \n",
       "3                   50.92              21.81           27.27   \n",
       "4                   64.35              19.87           15.78   \n",
       "...                   ...                ...             ...   \n",
       "4052589             29.17              33.33           37.50   \n",
       "4052590                 *                  *               *   \n",
       "4052591             32.92              25.10           41.98   \n",
       "4052592                 *                  *               *   \n",
       "4052593             35.53              26.40           38.07   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     410290                17.47               57.69   \n",
       "1                     208998                15.88               57.23   \n",
       "2                     201212                19.12               58.17   \n",
       "3                     316516                21.56               59.05   \n",
       "4                      27760                30.24               58.31   \n",
       "...                      ...                  ...                 ...   \n",
       "4052589                   24                    *                   *   \n",
       "4052590                    8                    *                   *   \n",
       "4052591                  243                10.29               47.74   \n",
       "4052592                    8                    *                   *   \n",
       "4052593                  197                 9.14               50.76   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      24.84                16.18               50.24   \n",
       "1                      26.89                13.96               49.35   \n",
       "2                      22.71                18.48               51.16   \n",
       "3                      19.39                19.88               52.83   \n",
       "4                      11.45                28.50               54.52   \n",
       "...                      ...                  ...                 ...   \n",
       "4052589                    *                    *                   *   \n",
       "4052590                    *                    *                   *   \n",
       "4052591                41.98                15.23               64.20   \n",
       "4052592                    *                    *                   *   \n",
       "4052593                40.10                16.75               64.47   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      33.58                11.75               70.96   \n",
       "1                      36.69                11.47               70.55   \n",
       "2                      30.36                12.04               71.39   \n",
       "3                      27.29                14.03               71.81   \n",
       "4                      16.98                19.43               71.38   \n",
       "...                      ...                  ...                 ...   \n",
       "4052589                    *                    *                   *   \n",
       "4052590                    *                    *                   *   \n",
       "4052591                20.58                12.76               66.67   \n",
       "4052592                    *                    *                   *   \n",
       "4052593                18.78                13.71               68.53   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      17.29                16.07               60.00   \n",
       "1                      17.99                15.20               59.33   \n",
       "2                      16.57                16.96               60.69   \n",
       "3                      14.16                19.54               61.39   \n",
       "4                       9.19                28.13               60.23   \n",
       "...                      ...                  ...                 ...   \n",
       "4052589                    *                    *                   *   \n",
       "4052590                    *                    *                   *   \n",
       "4052591                20.58                 0.00                0.00   \n",
       "4052592                    *                    *                   *   \n",
       "4052593                17.77                 0.00                0.00   \n",
       "\n",
       "        area_4_pct_below_std type_id  \n",
       "0                      23.94       4  \n",
       "1                      25.47       4  \n",
       "2                      22.35       4  \n",
       "3                      19.08       4  \n",
       "4                      11.65       4  \n",
       "...                      ...     ...  \n",
       "4052589                    *       7  \n",
       "4052590                    *       7  \n",
       "4052591                 0.00       7  \n",
       "4052592                    *       7  \n",
       "4052593                 0.00       7  \n",
       "\n",
       "[4052594 rows x 33 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sb_ca2023 = pd.read_csv('caaspp_csv_files_exported/sb_ca2023_all_csv_v1.csv', encoding='ISO-8859-1',dtype=str)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "sb_ca2023.columns = [\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'filler',\n",
    "    'test_year',\n",
    "    'student_group_id',\n",
    "    'test_type',\n",
    "    'total_tested_reporting_lvl',\n",
    "    'total_tested_with_scores',\n",
    "    'grade',\n",
    "    'test_id',\n",
    "    'students_enrolled',\n",
    "    'students_tested',\n",
    "    'mean_scale_score',\n",
    "    'pct_std_exceeded',\n",
    "    'pct_std_met',\n",
    "    'pct_std_met_above',\n",
    "    'pct_std_nearly_met',\n",
    "    'pct_std_not_met',\n",
    "    'students_with_scores',\n",
    "    'area_1_pct_above_std',\n",
    "    'area_1_pct_near_std',\n",
    "    'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std',\n",
    "    'area_2_pct_near_std',\n",
    "    'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std',\n",
    "    'area_3_pct_near_std',\n",
    "    'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std',\n",
    "    'area_4_pct_near_std',\n",
    "    'area_4_pct_below_std',\n",
    "    'type_id'\n",
    "]\n",
    "\n",
    "sb_ca2023"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_year</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>912</th>\n",
       "      <td>2023</td>\n",
       "      <td>01</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1822</th>\n",
       "      <td>2023</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2148</th>\n",
       "      <td>2023</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0112607</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2548</th>\n",
       "      <td>2023</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0123968</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4051692</th>\n",
       "      <td>2023</td>\n",
       "      <td>58</td>\n",
       "      <td>72751</td>\n",
       "      <td>6118806</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4051916</th>\n",
       "      <td>2023</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052156</th>\n",
       "      <td>2023</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0123570</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052258</th>\n",
       "      <td>2023</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0133751</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052382</th>\n",
       "      <td>2023</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11302 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        test_year county_code district_code school_code type_id\n",
       "0            2023          00         00000     0000000       4\n",
       "912          2023          01         00000     0000000       5\n",
       "1822         2023          01         10017     0000000       6\n",
       "2148         2023          01         10017     0112607       9\n",
       "2548         2023          01         10017     0123968       9\n",
       "...           ...         ...           ...         ...     ...\n",
       "4051692      2023          58         72751     6118806      10\n",
       "4051916      2023          58         72769     0000000       6\n",
       "4052156      2023          58         72769     0123570       7\n",
       "4052258      2023          58         72769     0133751       7\n",
       "4052382      2023          58         72769     5838305       7\n",
       "\n",
       "[11302 rows x 5 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This step is created to build a type_id dataset, which will be used to identify the type_id for entities in previous datasets (<= 2019)\n",
    "\n",
    "# Step 1: Filter for the current year\n",
    "filtered_df = sb_ca2023[(sb_ca2023['test_year'] == '2023')]\n",
    "\n",
    "# Step 2: Select distinct rows based on 'school_code' and 'type_id'\n",
    "df_type_id_23 = filtered_df[['test_year','county_code','district_code', 'school_code', 'type_id']].drop_duplicates()\n",
    "\n",
    "df_type_id_23"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2969670</td>\n",
       "      <td>2965421</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>424979</td>\n",
       "      <td>413597</td>\n",
       "      <td>2408.6</td>\n",
       "      <td>22.77</td>\n",
       "      <td>19.40</td>\n",
       "      <td>42.17</td>\n",
       "      <td>22.58</td>\n",
       "      <td>35.25</td>\n",
       "      <td>413308</td>\n",
       "      <td>18.13</td>\n",
       "      <td>57.32</td>\n",
       "      <td>24.56</td>\n",
       "      <td>16.02</td>\n",
       "      <td>48.71</td>\n",
       "      <td>35.27</td>\n",
       "      <td>11.77</td>\n",
       "      <td>71.11</td>\n",
       "      <td>17.12</td>\n",
       "      <td>16.30</td>\n",
       "      <td>59.57</td>\n",
       "      <td>24.13</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1518659</td>\n",
       "      <td>1516461</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>216959</td>\n",
       "      <td>210787</td>\n",
       "      <td>2400.7</td>\n",
       "      <td>20.58</td>\n",
       "      <td>18.53</td>\n",
       "      <td>39.11</td>\n",
       "      <td>22.49</td>\n",
       "      <td>38.40</td>\n",
       "      <td>210643</td>\n",
       "      <td>16.59</td>\n",
       "      <td>56.69</td>\n",
       "      <td>26.72</td>\n",
       "      <td>13.75</td>\n",
       "      <td>47.63</td>\n",
       "      <td>38.62</td>\n",
       "      <td>11.36</td>\n",
       "      <td>70.67</td>\n",
       "      <td>17.97</td>\n",
       "      <td>15.36</td>\n",
       "      <td>58.67</td>\n",
       "      <td>25.97</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1449399</td>\n",
       "      <td>1447353</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>207967</td>\n",
       "      <td>202760</td>\n",
       "      <td>2416.8</td>\n",
       "      <td>25.04</td>\n",
       "      <td>20.30</td>\n",
       "      <td>45.34</td>\n",
       "      <td>22.68</td>\n",
       "      <td>31.98</td>\n",
       "      <td>202615</td>\n",
       "      <td>19.72</td>\n",
       "      <td>57.97</td>\n",
       "      <td>22.31</td>\n",
       "      <td>18.38</td>\n",
       "      <td>49.82</td>\n",
       "      <td>31.79</td>\n",
       "      <td>12.19</td>\n",
       "      <td>71.56</td>\n",
       "      <td>16.24</td>\n",
       "      <td>17.28</td>\n",
       "      <td>60.51</td>\n",
       "      <td>22.21</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>6</td>\n",
       "      <td>B</td>\n",
       "      <td>2439001</td>\n",
       "      <td>2435619</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>321502</td>\n",
       "      <td>314456</td>\n",
       "      <td>2427.4</td>\n",
       "      <td>28.30</td>\n",
       "      <td>21.84</td>\n",
       "      <td>50.14</td>\n",
       "      <td>22.12</td>\n",
       "      <td>27.74</td>\n",
       "      <td>314244</td>\n",
       "      <td>22.43</td>\n",
       "      <td>58.42</td>\n",
       "      <td>19.15</td>\n",
       "      <td>19.74</td>\n",
       "      <td>51.30</td>\n",
       "      <td>28.96</td>\n",
       "      <td>14.16</td>\n",
       "      <td>71.80</td>\n",
       "      <td>14.04</td>\n",
       "      <td>19.91</td>\n",
       "      <td>60.88</td>\n",
       "      <td>19.21</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>7</td>\n",
       "      <td>B</td>\n",
       "      <td>132045</td>\n",
       "      <td>131948</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>25859</td>\n",
       "      <td>25591</td>\n",
       "      <td>2460.4</td>\n",
       "      <td>39.61</td>\n",
       "      <td>23.37</td>\n",
       "      <td>62.98</td>\n",
       "      <td>20.40</td>\n",
       "      <td>16.62</td>\n",
       "      <td>25575</td>\n",
       "      <td>31.05</td>\n",
       "      <td>57.30</td>\n",
       "      <td>11.64</td>\n",
       "      <td>28.70</td>\n",
       "      <td>52.27</td>\n",
       "      <td>19.04</td>\n",
       "      <td>19.50</td>\n",
       "      <td>71.27</td>\n",
       "      <td>9.23</td>\n",
       "      <td>28.23</td>\n",
       "      <td>59.92</td>\n",
       "      <td>11.86</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855776</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>223</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855777</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>224</td>\n",
       "      <td>B</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "      <td>19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>15.79</td>\n",
       "      <td>15.79</td>\n",
       "      <td>10.53</td>\n",
       "      <td>73.68</td>\n",
       "      <td>19</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855778</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>226</td>\n",
       "      <td>B</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>13.33</td>\n",
       "      <td>13.33</td>\n",
       "      <td>30.00</td>\n",
       "      <td>56.67</td>\n",
       "      <td>30</td>\n",
       "      <td>3.33</td>\n",
       "      <td>46.67</td>\n",
       "      <td>50.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>70.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>6.67</td>\n",
       "      <td>60.00</td>\n",
       "      <td>33.33</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855779</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>8</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855780</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022</td>\n",
       "      <td>241</td>\n",
       "      <td>B</td>\n",
       "      <td>218</td>\n",
       "      <td>218</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>239</td>\n",
       "      <td>218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.75</td>\n",
       "      <td>16.51</td>\n",
       "      <td>19.27</td>\n",
       "      <td>19.72</td>\n",
       "      <td>61.01</td>\n",
       "      <td>218</td>\n",
       "      <td>5.50</td>\n",
       "      <td>40.83</td>\n",
       "      <td>53.67</td>\n",
       "      <td>5.96</td>\n",
       "      <td>63.30</td>\n",
       "      <td>30.73</td>\n",
       "      <td>4.13</td>\n",
       "      <td>64.68</td>\n",
       "      <td>31.19</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3855781 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2022   \n",
       "1                00         00000     0000000    NaN      2022   \n",
       "2                00         00000     0000000    NaN      2022   \n",
       "3                00         00000     0000000    NaN      2022   \n",
       "4                00         00000     0000000    NaN      2022   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3855776          58         72769     5838305    NaN      2022   \n",
       "3855777          58         72769     5838305    NaN      2022   \n",
       "3855778          58         72769     5838305    NaN      2022   \n",
       "3855779          58         72769     5838305    NaN      2022   \n",
       "3855780          58         72769     5838305    NaN      2022   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    2969670   \n",
       "1                      3         B                    1518659   \n",
       "2                      4         B                    1449399   \n",
       "3                      6         B                    2439001   \n",
       "4                      7         B                     132045   \n",
       "...                  ...       ...                        ...   \n",
       "3855776              223         B                          *   \n",
       "3855777              224         B                         19   \n",
       "3855778              226         B                         30   \n",
       "3855779              227         B                          8   \n",
       "3855780              241         B                        218   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        2965421     3       1            424979   \n",
       "1                        1516461     3       1            216959   \n",
       "2                        1447353     3       1            207967   \n",
       "3                        2435619     3       1            321502   \n",
       "4                         131948     3       1             25859   \n",
       "...                          ...   ...     ...               ...   \n",
       "3855776                        *    13       2                 *   \n",
       "3855777                       19    13       2                21   \n",
       "3855778                       30    13       2                35   \n",
       "3855779                        8    13       2                 8   \n",
       "3855780                      218    13       2               239   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                413597           2408.6            22.77       19.40   \n",
       "1                210787           2400.7            20.58       18.53   \n",
       "2                202760           2416.8            25.04       20.30   \n",
       "3                314456           2427.4            28.30       21.84   \n",
       "4                 25591           2460.4            39.61       23.37   \n",
       "...                 ...              ...              ...         ...   \n",
       "3855776               *              NaN                *           *   \n",
       "3855777              19              NaN             0.00       15.79   \n",
       "3855778              30              NaN             0.00       13.33   \n",
       "3855779               8              NaN                *           *   \n",
       "3855780             218              NaN             2.75       16.51   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   42.17              22.58           35.25   \n",
       "1                   39.11              22.49           38.40   \n",
       "2                   45.34              22.68           31.98   \n",
       "3                   50.14              22.12           27.74   \n",
       "4                   62.98              20.40           16.62   \n",
       "...                   ...                ...             ...   \n",
       "3855776                 *                  *               *   \n",
       "3855777             15.79              10.53           73.68   \n",
       "3855778             13.33              30.00           56.67   \n",
       "3855779                 *                  *               *   \n",
       "3855780             19.27              19.72           61.01   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     413308                18.13               57.32   \n",
       "1                     210643                16.59               56.69   \n",
       "2                     202615                19.72               57.97   \n",
       "3                     314244                22.43               58.42   \n",
       "4                      25575                31.05               57.30   \n",
       "...                      ...                  ...                 ...   \n",
       "3855776                    *                    *                   *   \n",
       "3855777                   19                    *                   *   \n",
       "3855778                   30                 3.33               46.67   \n",
       "3855779                    8                    *                   *   \n",
       "3855780                  218                 5.50               40.83   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      24.56                16.02               48.71   \n",
       "1                      26.72                13.75               47.63   \n",
       "2                      22.31                18.38               49.82   \n",
       "3                      19.15                19.74               51.30   \n",
       "4                      11.64                28.70               52.27   \n",
       "...                      ...                  ...                 ...   \n",
       "3855776                    *                    *                   *   \n",
       "3855777                    *                    *                   *   \n",
       "3855778                50.00                10.00               70.00   \n",
       "3855779                    *                    *                   *   \n",
       "3855780                53.67                 5.96               63.30   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      35.27                11.77               71.11   \n",
       "1                      38.62                11.36               70.67   \n",
       "2                      31.79                12.19               71.56   \n",
       "3                      28.96                14.16               71.80   \n",
       "4                      19.04                19.50               71.27   \n",
       "...                      ...                  ...                 ...   \n",
       "3855776                    *                    *                   *   \n",
       "3855777                    *                    *                   *   \n",
       "3855778                20.00                 6.67               60.00   \n",
       "3855779                    *                    *                   *   \n",
       "3855780                30.73                 4.13               64.68   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      17.12                16.30               59.57   \n",
       "1                      17.97                15.36               58.67   \n",
       "2                      16.24                17.28               60.51   \n",
       "3                      14.04                19.91               60.88   \n",
       "4                       9.23                28.23               59.92   \n",
       "...                      ...                  ...                 ...   \n",
       "3855776                    *                    *                   *   \n",
       "3855777                    *                    *                   *   \n",
       "3855778                33.33                 0.00                0.00   \n",
       "3855779                    *                    *                   *   \n",
       "3855780                31.19                 0.00                0.00   \n",
       "\n",
       "        area_4_pct_below_std type_id  \n",
       "0                      24.13       4  \n",
       "1                      25.97       4  \n",
       "2                      22.21       4  \n",
       "3                      19.21       4  \n",
       "4                      11.86       4  \n",
       "...                      ...     ...  \n",
       "3855776                    *       7  \n",
       "3855777                    *       7  \n",
       "3855778                 0.00       7  \n",
       "3855779                    *       7  \n",
       "3855780                 0.00       7  \n",
       "\n",
       "[3855781 rows x 33 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sb_ca2022 = pd.read_csv('caaspp_csv_files_exported/sb_ca2022_all_csv_v1.csv', encoding='ISO-8859-1',dtype=str)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "sb_ca2022.columns = [\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'filler',\n",
    "    'test_year',\n",
    "    'student_group_id',\n",
    "    'test_type',\n",
    "    'total_tested_reporting_lvl',\n",
    "    'total_tested_with_scores',\n",
    "    'grade',\n",
    "    'test_id',\n",
    "    'students_enrolled',\n",
    "    'students_tested',\n",
    "    'mean_scale_score',\n",
    "    'pct_std_exceeded',\n",
    "    'pct_std_met',\n",
    "    'pct_std_met_above',\n",
    "    'pct_std_nearly_met',\n",
    "    'pct_std_not_met',\n",
    "    'students_with_scores',\n",
    "    'area_1_pct_above_std',\n",
    "    'area_1_pct_near_std',\n",
    "    'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std',\n",
    "    'area_2_pct_near_std',\n",
    "    'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std',\n",
    "    'area_3_pct_near_std',\n",
    "    'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std',\n",
    "    'area_4_pct_near_std',\n",
    "    'area_4_pct_below_std',\n",
    "    'type_id'\n",
    "]\n",
    "\n",
    "sb_ca2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_year</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868</th>\n",
       "      <td>2022</td>\n",
       "      <td>01</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1736</th>\n",
       "      <td>2022</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2002</th>\n",
       "      <td>2022</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0112607</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2486</th>\n",
       "      <td>2022</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0123968</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3854909</th>\n",
       "      <td>2022</td>\n",
       "      <td>58</td>\n",
       "      <td>72751</td>\n",
       "      <td>6118806</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855071</th>\n",
       "      <td>2022</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855319</th>\n",
       "      <td>2022</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0123570</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855453</th>\n",
       "      <td>2022</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0133751</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3855581</th>\n",
       "      <td>2022</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11339 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        test_year county_code district_code school_code type_id\n",
       "0            2022          00         00000     0000000       4\n",
       "868          2022          01         00000     0000000       5\n",
       "1736         2022          01         10017     0000000       6\n",
       "2002         2022          01         10017     0112607       9\n",
       "2486         2022          01         10017     0123968       9\n",
       "...           ...         ...           ...         ...     ...\n",
       "3854909      2022          58         72751     6118806      10\n",
       "3855071      2022          58         72769     0000000       6\n",
       "3855319      2022          58         72769     0123570       7\n",
       "3855453      2022          58         72769     0133751       7\n",
       "3855581      2022          58         72769     5838305       7\n",
       "\n",
       "[11339 rows x 5 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This step is created to build a type_id dataset, which will be used to identify the type_id for entities in previous datasets (<= 2019)\n",
    "\n",
    "# Step 1: Filter for the current year\n",
    "filtered_df = sb_ca2022[(sb_ca2022['test_year'] == '2022')]\n",
    "\n",
    "# Step 2: Select distinct rows based on 'school_code' and 'type_id'\n",
    "df_type_id_22 = filtered_df[['test_year','county_code','district_code', 'school_code', 'type_id']].drop_duplicates()\n",
    "\n",
    "df_type_id_22"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>743626</td>\n",
       "      <td>736192</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>432992</td>\n",
       "      <td>80311</td>\n",
       "      <td>2403.3</td>\n",
       "      <td>20.16</td>\n",
       "      <td>19.63</td>\n",
       "      <td>39.79</td>\n",
       "      <td>23.78</td>\n",
       "      <td>36.43</td>\n",
       "      <td>80031</td>\n",
       "      <td>16.67</td>\n",
       "      <td>55.44</td>\n",
       "      <td>27.89</td>\n",
       "      <td>12.98</td>\n",
       "      <td>50.72</td>\n",
       "      <td>36.30</td>\n",
       "      <td>10.49</td>\n",
       "      <td>71.78</td>\n",
       "      <td>17.72</td>\n",
       "      <td>16.57</td>\n",
       "      <td>61.65</td>\n",
       "      <td>21.77</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>375522</td>\n",
       "      <td>371742</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>221170</td>\n",
       "      <td>40659</td>\n",
       "      <td>2395.9</td>\n",
       "      <td>18.11</td>\n",
       "      <td>18.78</td>\n",
       "      <td>36.89</td>\n",
       "      <td>23.80</td>\n",
       "      <td>39.31</td>\n",
       "      <td>40527</td>\n",
       "      <td>15.23</td>\n",
       "      <td>54.79</td>\n",
       "      <td>29.98</td>\n",
       "      <td>10.93</td>\n",
       "      <td>49.51</td>\n",
       "      <td>39.57</td>\n",
       "      <td>9.89</td>\n",
       "      <td>71.55</td>\n",
       "      <td>18.56</td>\n",
       "      <td>15.53</td>\n",
       "      <td>61.02</td>\n",
       "      <td>23.45</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>367928</td>\n",
       "      <td>364279</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>211800</td>\n",
       "      <td>39648</td>\n",
       "      <td>2410.8</td>\n",
       "      <td>22.27</td>\n",
       "      <td>20.51</td>\n",
       "      <td>42.77</td>\n",
       "      <td>23.74</td>\n",
       "      <td>33.48</td>\n",
       "      <td>39500</td>\n",
       "      <td>18.15</td>\n",
       "      <td>56.10</td>\n",
       "      <td>25.75</td>\n",
       "      <td>15.08</td>\n",
       "      <td>51.97</td>\n",
       "      <td>32.95</td>\n",
       "      <td>11.11</td>\n",
       "      <td>72.02</td>\n",
       "      <td>16.86</td>\n",
       "      <td>17.64</td>\n",
       "      <td>62.30</td>\n",
       "      <td>20.06</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>6</td>\n",
       "      <td>B</td>\n",
       "      <td>628259</td>\n",
       "      <td>622358</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>320066</td>\n",
       "      <td>60353</td>\n",
       "      <td>2422.5</td>\n",
       "      <td>25.22</td>\n",
       "      <td>22.70</td>\n",
       "      <td>47.92</td>\n",
       "      <td>23.93</td>\n",
       "      <td>28.15</td>\n",
       "      <td>60165</td>\n",
       "      <td>20.74</td>\n",
       "      <td>57.81</td>\n",
       "      <td>21.45</td>\n",
       "      <td>16.21</td>\n",
       "      <td>54.66</td>\n",
       "      <td>29.13</td>\n",
       "      <td>12.80</td>\n",
       "      <td>72.67</td>\n",
       "      <td>14.52</td>\n",
       "      <td>20.29</td>\n",
       "      <td>62.62</td>\n",
       "      <td>17.10</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>7</td>\n",
       "      <td>B</td>\n",
       "      <td>31747</td>\n",
       "      <td>31503</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>15481</td>\n",
       "      <td>2916</td>\n",
       "      <td>2481.4</td>\n",
       "      <td>48.25</td>\n",
       "      <td>23.52</td>\n",
       "      <td>71.77</td>\n",
       "      <td>16.96</td>\n",
       "      <td>11.26</td>\n",
       "      <td>2912</td>\n",
       "      <td>38.60</td>\n",
       "      <td>52.32</td>\n",
       "      <td>9.08</td>\n",
       "      <td>33.84</td>\n",
       "      <td>52.05</td>\n",
       "      <td>14.11</td>\n",
       "      <td>23.18</td>\n",
       "      <td>69.33</td>\n",
       "      <td>7.50</td>\n",
       "      <td>37.74</td>\n",
       "      <td>55.39</td>\n",
       "      <td>6.87</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586706</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>223</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586707</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>224</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586708</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>225</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586709</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>226</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586710</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3586711 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2021   \n",
       "1                00         00000     0000000    NaN      2021   \n",
       "2                00         00000     0000000    NaN      2021   \n",
       "3                00         00000     0000000    NaN      2021   \n",
       "4                00         00000     0000000    NaN      2021   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3586706          58         72769     5838305    NaN      2021   \n",
       "3586707          58         72769     5838305    NaN      2021   \n",
       "3586708          58         72769     5838305    NaN      2021   \n",
       "3586709          58         72769     5838305    NaN      2021   \n",
       "3586710          58         72769     5838305    NaN      2021   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                     743626   \n",
       "1                      3         B                     375522   \n",
       "2                      4         B                     367928   \n",
       "3                      6         B                     628259   \n",
       "4                      7         B                      31747   \n",
       "...                  ...       ...                        ...   \n",
       "3586706              223         B                          0   \n",
       "3586707              224         B                          0   \n",
       "3586708              225         B                          0   \n",
       "3586709              226         B                          0   \n",
       "3586710              227         B                          0   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                         736192     3       1            432992   \n",
       "1                         371742     3       1            221170   \n",
       "2                         364279     3       1            211800   \n",
       "3                         622358     3       1            320066   \n",
       "4                          31503     3       1             15481   \n",
       "...                          ...   ...     ...               ...   \n",
       "3586706                        0    13       2                 *   \n",
       "3586707                        0    13       2                23   \n",
       "3586708                        0    13       2                 *   \n",
       "3586709                        0    13       2                56   \n",
       "3586710                        0    13       2                 9   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                 80311           2403.3            20.16       19.63   \n",
       "1                 40659           2395.9            18.11       18.78   \n",
       "2                 39648           2410.8            22.27       20.51   \n",
       "3                 60353           2422.5            25.22       22.70   \n",
       "4                  2916           2481.4            48.25       23.52   \n",
       "...                 ...              ...              ...         ...   \n",
       "3586706               0              NaN              NaN         NaN   \n",
       "3586707               0              NaN              NaN         NaN   \n",
       "3586708               0              NaN              NaN         NaN   \n",
       "3586709               0              NaN              NaN         NaN   \n",
       "3586710               0              NaN              NaN         NaN   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   39.79              23.78           36.43   \n",
       "1                   36.89              23.80           39.31   \n",
       "2                   42.77              23.74           33.48   \n",
       "3                   47.92              23.93           28.15   \n",
       "4                   71.77              16.96           11.26   \n",
       "...                   ...                ...             ...   \n",
       "3586706               NaN                NaN             NaN   \n",
       "3586707               NaN                NaN             NaN   \n",
       "3586708               NaN                NaN             NaN   \n",
       "3586709               NaN                NaN             NaN   \n",
       "3586710               NaN                NaN             NaN   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                      80031                16.67               55.44   \n",
       "1                      40527                15.23               54.79   \n",
       "2                      39500                18.15               56.10   \n",
       "3                      60165                20.74               57.81   \n",
       "4                       2912                38.60               52.32   \n",
       "...                      ...                  ...                 ...   \n",
       "3586706                    0                  NaN                 NaN   \n",
       "3586707                    0                  NaN                 NaN   \n",
       "3586708                    0                  NaN                 NaN   \n",
       "3586709                    0                  NaN                 NaN   \n",
       "3586710                    0                  NaN                 NaN   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      27.89                12.98               50.72   \n",
       "1                      29.98                10.93               49.51   \n",
       "2                      25.75                15.08               51.97   \n",
       "3                      21.45                16.21               54.66   \n",
       "4                       9.08                33.84               52.05   \n",
       "...                      ...                  ...                 ...   \n",
       "3586706                  NaN                  NaN                 NaN   \n",
       "3586707                  NaN                  NaN                 NaN   \n",
       "3586708                  NaN                  NaN                 NaN   \n",
       "3586709                  NaN                  NaN                 NaN   \n",
       "3586710                  NaN                  NaN                 NaN   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      36.30                10.49               71.78   \n",
       "1                      39.57                 9.89               71.55   \n",
       "2                      32.95                11.11               72.02   \n",
       "3                      29.13                12.80               72.67   \n",
       "4                      14.11                23.18               69.33   \n",
       "...                      ...                  ...                 ...   \n",
       "3586706                  NaN                  NaN                 NaN   \n",
       "3586707                  NaN                  NaN                 NaN   \n",
       "3586708                  NaN                  NaN                 NaN   \n",
       "3586709                  NaN                  NaN                 NaN   \n",
       "3586710                  NaN                  NaN                 NaN   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      17.72                16.57               61.65   \n",
       "1                      18.56                15.53               61.02   \n",
       "2                      16.86                17.64               62.30   \n",
       "3                      14.52                20.29               62.62   \n",
       "4                       7.50                37.74               55.39   \n",
       "...                      ...                  ...                 ...   \n",
       "3586706                  NaN                  NaN                 NaN   \n",
       "3586707                  NaN                  NaN                 NaN   \n",
       "3586708                  NaN                  NaN                 NaN   \n",
       "3586709                  NaN                  NaN                 NaN   \n",
       "3586710                  NaN                  NaN                 NaN   \n",
       "\n",
       "        area_4_pct_below_std type_id  \n",
       "0                      21.77       4  \n",
       "1                      23.45       4  \n",
       "2                      20.06       4  \n",
       "3                      17.10       4  \n",
       "4                       6.87       4  \n",
       "...                      ...     ...  \n",
       "3586706                  NaN       7  \n",
       "3586707                  NaN       7  \n",
       "3586708                  NaN       7  \n",
       "3586709                  NaN       7  \n",
       "3586710                  NaN       7  \n",
       "\n",
       "[3586711 rows x 33 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sb_ca2021 = pd.read_csv('caaspp_csv_files_exported/sb_ca2021_all_csv_v2.csv', encoding='ISO-8859-1',dtype=str)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "sb_ca2021.columns = [\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'filler',\n",
    "    'test_year',\n",
    "    'student_group_id',\n",
    "    'test_type',\n",
    "    'total_tested_reporting_lvl',\n",
    "    'total_tested_with_scores',\n",
    "    'grade',\n",
    "    'test_id',\n",
    "    'students_enrolled',\n",
    "    'students_tested',\n",
    "    'mean_scale_score',\n",
    "    'pct_std_exceeded',\n",
    "    'pct_std_met',\n",
    "    'pct_std_met_above',\n",
    "    'pct_std_nearly_met',\n",
    "    'pct_std_not_met',\n",
    "    'students_with_scores',\n",
    "    'area_1_pct_above_std',\n",
    "    'area_1_pct_near_std',\n",
    "    'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std',\n",
    "    'area_2_pct_near_std',\n",
    "    'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std',\n",
    "    'area_3_pct_near_std',\n",
    "    'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std',\n",
    "    'area_4_pct_near_std',\n",
    "    'area_4_pct_below_std',\n",
    "    'type_id'\n",
    "]\n",
    "\n",
    "sb_ca2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_year</th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021</td>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>2021</td>\n",
       "      <td>01</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1672</th>\n",
       "      <td>2021</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1900</th>\n",
       "      <td>2021</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0112607</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2166</th>\n",
       "      <td>2021</td>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0124172</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3585907</th>\n",
       "      <td>2021</td>\n",
       "      <td>58</td>\n",
       "      <td>72751</td>\n",
       "      <td>6118806</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586139</th>\n",
       "      <td>2021</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586347</th>\n",
       "      <td>2021</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0123570</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586395</th>\n",
       "      <td>2021</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0133751</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586527</th>\n",
       "      <td>2021</td>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11149 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        test_year county_code district_code school_code type_id\n",
       "0            2021          00         00000     0000000       4\n",
       "836          2021          01         00000     0000000       5\n",
       "1672         2021          01         10017     0000000       6\n",
       "1900         2021          01         10017     0112607       9\n",
       "2166         2021          01         10017     0124172       9\n",
       "...           ...         ...           ...         ...     ...\n",
       "3585907      2021          58         72751     6118806      10\n",
       "3586139      2021          58         72769     0000000       6\n",
       "3586347      2021          58         72769     0123570       7\n",
       "3586395      2021          58         72769     0133751       7\n",
       "3586527      2021          58         72769     5838305       7\n",
       "\n",
       "[11149 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This step is created to build a type_id dataset, which will be used to identify the type_id for entities in previous datasets (<= 2019)\n",
    "\n",
    "# Step 1: Filter for the current year\n",
    "filtered_df = sb_ca2021[(sb_ca2021['test_year'] == '2021')]\n",
    "\n",
    "# Step 2: Select distinct rows based on 'school_code' and 'type_id'\n",
    "df_type_id_21 = filtered_df[['test_year', 'county_code','district_code', 'school_code', 'type_id']].drop_duplicates()\n",
    "\n",
    "df_type_id_21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1: Concatenate the DataFrames\n",
    "df_type_id = pd.concat([df_type_id_23, df_type_id_22, df_type_id_21])\n",
    "\n",
    "# Step 2: Convert 'test_year' to a numeric format for proper comparison (skip this step if 'test_year' is already numeric)\n",
    "df_type_id['test_year'] = pd.to_numeric(df_type_id['test_year'])\n",
    "\n",
    "# Step 3: Sort the DataFrame by 'county_code', 'district_code', 'school_code', and 'test_year' in descending order\n",
    "df_type_id = df_type_id.sort_values(by=['county_code', 'district_code', 'school_code', 'test_year'], ascending=[True, True, True, False])\n",
    "\n",
    "# Step 4: Drop duplicates based on 'county_code', 'district_code', and 'school_code'\n",
    "df_type_id = df_type_id.drop_duplicates(subset=['county_code', 'district_code', 'school_code'])\n",
    "\n",
    "# Step 5: Select only the desired columns\n",
    "df_type_id = df_type_id[['county_code', 'district_code', 'school_code', 'type_id']]\n",
    "\n",
    "# The resulting df_type_id DataFrame will contain only the rows with the highest 'test_year' for each combination of 'county_code', 'district_code', and 'school_code', along with all other columns.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>912</th>\n",
       "      <td>01</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1822</th>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2148</th>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0112607</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2548</th>\n",
       "      <td>01</td>\n",
       "      <td>10017</td>\n",
       "      <td>0123968</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4051692</th>\n",
       "      <td>58</td>\n",
       "      <td>72751</td>\n",
       "      <td>6118806</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4051916</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052156</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0123570</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052258</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0133751</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052382</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11615 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code type_id\n",
       "0                00         00000     0000000       4\n",
       "912              01         00000     0000000       5\n",
       "1822             01         10017     0000000       6\n",
       "2148             01         10017     0112607       9\n",
       "2548             01         10017     0123968       9\n",
       "...             ...           ...         ...     ...\n",
       "4051692          58         72751     6118806      10\n",
       "4051916          58         72769     0000000       6\n",
       "4052156          58         72769     0123570       7\n",
       "4052258          58         72769     0133751       7\n",
       "4052382          58         72769     5838305       7\n",
       "\n",
       "[11615 rows x 4 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_type_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15</td>\n",
       "      <td>75630</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19</td>\n",
       "      <td>76497</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19</td>\n",
       "      <td>76547</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19</td>\n",
       "      <td>76885</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19</td>\n",
       "      <td>77073</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>33</td>\n",
       "      <td>76943</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>37</td>\n",
       "      <td>76901</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>38</td>\n",
       "      <td>76919</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>42</td>\n",
       "      <td>76950</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>52</td>\n",
       "      <td>71514</td>\n",
       "      <td>0000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>07</td>\n",
       "      <td>61796</td>\n",
       "      <td>0132233</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>07</td>\n",
       "      <td>61796</td>\n",
       "      <td>6118368</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   county_code district_code school_code type_id\n",
       "0           15         75630     0000000       6\n",
       "1           19         76497     0000000       6\n",
       "2           19         76547     0000000       6\n",
       "3           19         76885     0000000       6\n",
       "4           19         77073     0000000       6\n",
       "5           33         76943     0000000       6\n",
       "6           37         76901     0000000       6\n",
       "7           38         76919     0000000       6\n",
       "8           42         76950     0000000       6\n",
       "9           52         71514     0000000       6\n",
       "10          07         61796     0132233       9\n",
       "11          07         61796     6118368       9"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Missing Data Valuess\n",
    "data = {\n",
    "    'county_code': ['15', '19', '19', '19', '19', '33', '37', '38', '42', '52', '07', '07'],\n",
    "    'district_code': ['75630', '76497', '76547', '76885', '77073', '76943', '76901', '76919', '76950', '71514', '61796', '61796'],\n",
    "    'school_code': ['0000000', '0000000', '0000000', '0000000', '0000000', '0000000', '0000000', '0000000', '0000000', '0000000', '0132233', '6118368'],\n",
    "    'type_id': ['6', '6', '6', '6', '6', '6', '6', '6', '6', '6', '9', '9']\n",
    "}\n",
    "\n",
    "# Create DataFrame\n",
    "df_type_id_missing = pd.DataFrame(data)\n",
    "df_type_id_missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_type_id_final = pd.concat([df_type_id, df_type_id_missing])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3165580</td>\n",
       "      <td>3162910</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>453153</td>\n",
       "      <td>442858</td>\n",
       "      <td>2425.2</td>\n",
       "      <td>26.35</td>\n",
       "      <td>22.19</td>\n",
       "      <td>48.54</td>\n",
       "      <td>23.42</td>\n",
       "      <td>28.04</td>\n",
       "      <td>442660</td>\n",
       "      <td>26.64</td>\n",
       "      <td>45.35</td>\n",
       "      <td>28.01</td>\n",
       "      <td>20.83</td>\n",
       "      <td>49.66</td>\n",
       "      <td>29.50</td>\n",
       "      <td>21.89</td>\n",
       "      <td>61.25</td>\n",
       "      <td>16.86</td>\n",
       "      <td>24.14</td>\n",
       "      <td>48.16</td>\n",
       "      <td>27.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1616938</td>\n",
       "      <td>1615465</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>231186</td>\n",
       "      <td>225530</td>\n",
       "      <td>2417.0</td>\n",
       "      <td>23.86</td>\n",
       "      <td>21.16</td>\n",
       "      <td>45.02</td>\n",
       "      <td>23.54</td>\n",
       "      <td>31.44</td>\n",
       "      <td>225427</td>\n",
       "      <td>24.12</td>\n",
       "      <td>44.79</td>\n",
       "      <td>31.09</td>\n",
       "      <td>17.93</td>\n",
       "      <td>49.03</td>\n",
       "      <td>33.04</td>\n",
       "      <td>20.66</td>\n",
       "      <td>60.54</td>\n",
       "      <td>18.81</td>\n",
       "      <td>22.66</td>\n",
       "      <td>46.69</td>\n",
       "      <td>30.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1548642</td>\n",
       "      <td>1547445</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>221967</td>\n",
       "      <td>217328</td>\n",
       "      <td>2433.8</td>\n",
       "      <td>28.93</td>\n",
       "      <td>23.27</td>\n",
       "      <td>52.20</td>\n",
       "      <td>23.29</td>\n",
       "      <td>24.51</td>\n",
       "      <td>217233</td>\n",
       "      <td>29.26</td>\n",
       "      <td>45.93</td>\n",
       "      <td>24.81</td>\n",
       "      <td>23.85</td>\n",
       "      <td>50.32</td>\n",
       "      <td>25.83</td>\n",
       "      <td>23.17</td>\n",
       "      <td>61.99</td>\n",
       "      <td>14.84</td>\n",
       "      <td>25.68</td>\n",
       "      <td>49.69</td>\n",
       "      <td>24.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>6</td>\n",
       "      <td>B</td>\n",
       "      <td>2651676</td>\n",
       "      <td>2649540</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>352884</td>\n",
       "      <td>346840</td>\n",
       "      <td>2442.5</td>\n",
       "      <td>32.09</td>\n",
       "      <td>24.76</td>\n",
       "      <td>56.85</td>\n",
       "      <td>22.39</td>\n",
       "      <td>20.76</td>\n",
       "      <td>346684</td>\n",
       "      <td>32.26</td>\n",
       "      <td>46.50</td>\n",
       "      <td>21.24</td>\n",
       "      <td>25.12</td>\n",
       "      <td>51.92</td>\n",
       "      <td>22.96</td>\n",
       "      <td>26.17</td>\n",
       "      <td>61.23</td>\n",
       "      <td>12.60</td>\n",
       "      <td>28.93</td>\n",
       "      <td>49.45</td>\n",
       "      <td>21.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>7</td>\n",
       "      <td>B</td>\n",
       "      <td>132465</td>\n",
       "      <td>132405</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>16409</td>\n",
       "      <td>16255</td>\n",
       "      <td>2491.5</td>\n",
       "      <td>54.47</td>\n",
       "      <td>23.98</td>\n",
       "      <td>78.45</td>\n",
       "      <td>13.96</td>\n",
       "      <td>7.59</td>\n",
       "      <td>16252</td>\n",
       "      <td>52.57</td>\n",
       "      <td>38.73</td>\n",
       "      <td>8.70</td>\n",
       "      <td>42.74</td>\n",
       "      <td>47.23</td>\n",
       "      <td>10.03</td>\n",
       "      <td>42.25</td>\n",
       "      <td>53.14</td>\n",
       "      <td>4.60</td>\n",
       "      <td>47.75</td>\n",
       "      <td>43.21</td>\n",
       "      <td>9.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576486</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>222</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576487</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>223</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576488</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>224</td>\n",
       "      <td>B</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>15.79</td>\n",
       "      <td>15.79</td>\n",
       "      <td>42.11</td>\n",
       "      <td>42.11</td>\n",
       "      <td>19</td>\n",
       "      <td>5.26</td>\n",
       "      <td>36.84</td>\n",
       "      <td>57.89</td>\n",
       "      <td>0.00</td>\n",
       "      <td>63.16</td>\n",
       "      <td>36.84</td>\n",
       "      <td>5.26</td>\n",
       "      <td>57.89</td>\n",
       "      <td>36.84</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576489</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>226</td>\n",
       "      <td>B</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>57</td>\n",
       "      <td>54</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.41</td>\n",
       "      <td>24.07</td>\n",
       "      <td>31.48</td>\n",
       "      <td>29.63</td>\n",
       "      <td>38.89</td>\n",
       "      <td>54</td>\n",
       "      <td>22.22</td>\n",
       "      <td>22.22</td>\n",
       "      <td>55.56</td>\n",
       "      <td>16.67</td>\n",
       "      <td>44.44</td>\n",
       "      <td>38.89</td>\n",
       "      <td>14.81</td>\n",
       "      <td>57.41</td>\n",
       "      <td>27.78</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576490</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>9</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3576491 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2019   \n",
       "1                00         00000     0000000    NaN      2019   \n",
       "2                00         00000     0000000    NaN      2019   \n",
       "3                00         00000     0000000    NaN      2019   \n",
       "4                00         00000     0000000    NaN      2019   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3576486          58         72769     5838305    NaN      2019   \n",
       "3576487          58         72769     5838305    NaN      2019   \n",
       "3576488          58         72769     5838305    NaN      2019   \n",
       "3576489          58         72769     5838305    NaN      2019   \n",
       "3576490          58         72769     5838305    NaN      2019   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    3165580   \n",
       "1                      3         B                    1616938   \n",
       "2                      4         B                    1548642   \n",
       "3                      6         B                    2651676   \n",
       "4                      7         B                     132465   \n",
       "...                  ...       ...                        ...   \n",
       "3576486              222         B                          *   \n",
       "3576487              223         B                          *   \n",
       "3576488              224         B                         19   \n",
       "3576489              226         B                         54   \n",
       "3576490              227         B                          9   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        3162910     3       1            453153   \n",
       "1                        1615465     3       1            231186   \n",
       "2                        1547445     3       1            221967   \n",
       "3                        2649540     3       1            352884   \n",
       "4                         132405     3       1             16409   \n",
       "...                          ...   ...     ...               ...   \n",
       "3576486                        *    13       2                 *   \n",
       "3576487                        *    13       2                 *   \n",
       "3576488                       19    13       2                19   \n",
       "3576489                       54    13       2                57   \n",
       "3576490                        9    13       2                 9   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                442858           2425.2            26.35       22.19   \n",
       "1                225530           2417.0            23.86       21.16   \n",
       "2                217328           2433.8            28.93       23.27   \n",
       "3                346840           2442.5            32.09       24.76   \n",
       "4                 16255           2491.5            54.47       23.98   \n",
       "...                 ...              ...              ...         ...   \n",
       "3576486               *              NaN                *           *   \n",
       "3576487               *              NaN                *           *   \n",
       "3576488              19              NaN             0.00       15.79   \n",
       "3576489              54              NaN             7.41       24.07   \n",
       "3576490               9              NaN                *           *   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   48.54              23.42           28.04   \n",
       "1                   45.02              23.54           31.44   \n",
       "2                   52.20              23.29           24.51   \n",
       "3                   56.85              22.39           20.76   \n",
       "4                   78.45              13.96            7.59   \n",
       "...                   ...                ...             ...   \n",
       "3576486                 *                  *               *   \n",
       "3576487                 *                  *               *   \n",
       "3576488             15.79              42.11           42.11   \n",
       "3576489             31.48              29.63           38.89   \n",
       "3576490                 *                  *               *   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     442660                26.64               45.35   \n",
       "1                     225427                24.12               44.79   \n",
       "2                     217233                29.26               45.93   \n",
       "3                     346684                32.26               46.50   \n",
       "4                      16252                52.57               38.73   \n",
       "...                      ...                  ...                 ...   \n",
       "3576486                    *                    *                   *   \n",
       "3576487                    *                    *                   *   \n",
       "3576488                   19                 5.26               36.84   \n",
       "3576489                   54                22.22               22.22   \n",
       "3576490                    9                    *                   *   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      28.01                20.83               49.66   \n",
       "1                      31.09                17.93               49.03   \n",
       "2                      24.81                23.85               50.32   \n",
       "3                      21.24                25.12               51.92   \n",
       "4                       8.70                42.74               47.23   \n",
       "...                      ...                  ...                 ...   \n",
       "3576486                    *                    *                   *   \n",
       "3576487                    *                    *                   *   \n",
       "3576488                57.89                 0.00               63.16   \n",
       "3576489                55.56                16.67               44.44   \n",
       "3576490                    *                    *                   *   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      29.50                21.89               61.25   \n",
       "1                      33.04                20.66               60.54   \n",
       "2                      25.83                23.17               61.99   \n",
       "3                      22.96                26.17               61.23   \n",
       "4                      10.03                42.25               53.14   \n",
       "...                      ...                  ...                 ...   \n",
       "3576486                    *                    *                   *   \n",
       "3576487                    *                    *                   *   \n",
       "3576488                36.84                 5.26               57.89   \n",
       "3576489                38.89                14.81               57.41   \n",
       "3576490                    *                    *                   *   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      16.86                24.14               48.16   \n",
       "1                      18.81                22.66               46.69   \n",
       "2                      14.84                25.68               49.69   \n",
       "3                      12.60                28.93               49.45   \n",
       "4                       4.60                47.75               43.21   \n",
       "...                      ...                  ...                 ...   \n",
       "3576486                    *                    *                   *   \n",
       "3576487                    *                    *                   *   \n",
       "3576488                36.84                 0.00                0.00   \n",
       "3576489                27.78                 0.00                0.00   \n",
       "3576490                    *                    *                   *   \n",
       "\n",
       "        area_4_pct_below_std  \n",
       "0                      27.70  \n",
       "1                      30.65  \n",
       "2                      24.63  \n",
       "3                      21.62  \n",
       "4                       9.04  \n",
       "...                      ...  \n",
       "3576486                    *  \n",
       "3576487                    *  \n",
       "3576488                 0.00  \n",
       "3576489                 0.00  \n",
       "3576490                    *  \n",
       "\n",
       "[3576491 rows x 32 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sb_ca2019 = pd.read_csv('caaspp_csv_files_exported/sb_ca2019_all_csv_v4.csv', encoding='ISO-8859-1',dtype=str)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "sb_ca2019.columns = [\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'filler',\n",
    "    'test_year',\n",
    "    'student_group_id',\n",
    "    'test_type',\n",
    "    'total_tested_reporting_lvl',\n",
    "    'total_tested_with_scores',\n",
    "    'grade',\n",
    "    'test_id',\n",
    "    'students_enrolled',\n",
    "    'students_tested',\n",
    "    'mean_scale_score',\n",
    "    'pct_std_exceeded',\n",
    "    'pct_std_met',\n",
    "    'pct_std_met_above',\n",
    "    'pct_std_nearly_met',\n",
    "    'pct_std_not_met',\n",
    "    'students_with_scores',\n",
    "    'area_1_pct_above_std',\n",
    "    'area_1_pct_near_std',\n",
    "    'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std',\n",
    "    'area_2_pct_near_std',\n",
    "    'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std',\n",
    "    'area_3_pct_near_std',\n",
    "    'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std',\n",
    "    'area_4_pct_near_std',\n",
    "    'area_4_pct_below_std'\n",
    "]\n",
    "\n",
    "sb_ca2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3165580</td>\n",
       "      <td>3162910</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>453153</td>\n",
       "      <td>442858</td>\n",
       "      <td>2425.2</td>\n",
       "      <td>26.35</td>\n",
       "      <td>22.19</td>\n",
       "      <td>48.54</td>\n",
       "      <td>23.42</td>\n",
       "      <td>28.04</td>\n",
       "      <td>442660</td>\n",
       "      <td>26.64</td>\n",
       "      <td>45.35</td>\n",
       "      <td>28.01</td>\n",
       "      <td>20.83</td>\n",
       "      <td>49.66</td>\n",
       "      <td>29.50</td>\n",
       "      <td>21.89</td>\n",
       "      <td>61.25</td>\n",
       "      <td>16.86</td>\n",
       "      <td>24.14</td>\n",
       "      <td>48.16</td>\n",
       "      <td>27.70</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1616938</td>\n",
       "      <td>1615465</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>231186</td>\n",
       "      <td>225530</td>\n",
       "      <td>2417.0</td>\n",
       "      <td>23.86</td>\n",
       "      <td>21.16</td>\n",
       "      <td>45.02</td>\n",
       "      <td>23.54</td>\n",
       "      <td>31.44</td>\n",
       "      <td>225427</td>\n",
       "      <td>24.12</td>\n",
       "      <td>44.79</td>\n",
       "      <td>31.09</td>\n",
       "      <td>17.93</td>\n",
       "      <td>49.03</td>\n",
       "      <td>33.04</td>\n",
       "      <td>20.66</td>\n",
       "      <td>60.54</td>\n",
       "      <td>18.81</td>\n",
       "      <td>22.66</td>\n",
       "      <td>46.69</td>\n",
       "      <td>30.65</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1548642</td>\n",
       "      <td>1547445</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>221967</td>\n",
       "      <td>217328</td>\n",
       "      <td>2433.8</td>\n",
       "      <td>28.93</td>\n",
       "      <td>23.27</td>\n",
       "      <td>52.20</td>\n",
       "      <td>23.29</td>\n",
       "      <td>24.51</td>\n",
       "      <td>217233</td>\n",
       "      <td>29.26</td>\n",
       "      <td>45.93</td>\n",
       "      <td>24.81</td>\n",
       "      <td>23.85</td>\n",
       "      <td>50.32</td>\n",
       "      <td>25.83</td>\n",
       "      <td>23.17</td>\n",
       "      <td>61.99</td>\n",
       "      <td>14.84</td>\n",
       "      <td>25.68</td>\n",
       "      <td>49.69</td>\n",
       "      <td>24.63</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>6</td>\n",
       "      <td>B</td>\n",
       "      <td>2651676</td>\n",
       "      <td>2649540</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>352884</td>\n",
       "      <td>346840</td>\n",
       "      <td>2442.5</td>\n",
       "      <td>32.09</td>\n",
       "      <td>24.76</td>\n",
       "      <td>56.85</td>\n",
       "      <td>22.39</td>\n",
       "      <td>20.76</td>\n",
       "      <td>346684</td>\n",
       "      <td>32.26</td>\n",
       "      <td>46.50</td>\n",
       "      <td>21.24</td>\n",
       "      <td>25.12</td>\n",
       "      <td>51.92</td>\n",
       "      <td>22.96</td>\n",
       "      <td>26.17</td>\n",
       "      <td>61.23</td>\n",
       "      <td>12.60</td>\n",
       "      <td>28.93</td>\n",
       "      <td>49.45</td>\n",
       "      <td>21.62</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>7</td>\n",
       "      <td>B</td>\n",
       "      <td>132465</td>\n",
       "      <td>132405</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>16409</td>\n",
       "      <td>16255</td>\n",
       "      <td>2491.5</td>\n",
       "      <td>54.47</td>\n",
       "      <td>23.98</td>\n",
       "      <td>78.45</td>\n",
       "      <td>13.96</td>\n",
       "      <td>7.59</td>\n",
       "      <td>16252</td>\n",
       "      <td>52.57</td>\n",
       "      <td>38.73</td>\n",
       "      <td>8.70</td>\n",
       "      <td>42.74</td>\n",
       "      <td>47.23</td>\n",
       "      <td>10.03</td>\n",
       "      <td>42.25</td>\n",
       "      <td>53.14</td>\n",
       "      <td>4.60</td>\n",
       "      <td>47.75</td>\n",
       "      <td>43.21</td>\n",
       "      <td>9.04</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576486</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>222</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576487</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>223</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576488</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>224</td>\n",
       "      <td>B</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>15.79</td>\n",
       "      <td>15.79</td>\n",
       "      <td>42.11</td>\n",
       "      <td>42.11</td>\n",
       "      <td>19</td>\n",
       "      <td>5.26</td>\n",
       "      <td>36.84</td>\n",
       "      <td>57.89</td>\n",
       "      <td>0.00</td>\n",
       "      <td>63.16</td>\n",
       "      <td>36.84</td>\n",
       "      <td>5.26</td>\n",
       "      <td>57.89</td>\n",
       "      <td>36.84</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576489</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>226</td>\n",
       "      <td>B</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>57</td>\n",
       "      <td>54</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.41</td>\n",
       "      <td>24.07</td>\n",
       "      <td>31.48</td>\n",
       "      <td>29.63</td>\n",
       "      <td>38.89</td>\n",
       "      <td>54</td>\n",
       "      <td>22.22</td>\n",
       "      <td>22.22</td>\n",
       "      <td>55.56</td>\n",
       "      <td>16.67</td>\n",
       "      <td>44.44</td>\n",
       "      <td>38.89</td>\n",
       "      <td>14.81</td>\n",
       "      <td>57.41</td>\n",
       "      <td>27.78</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3576490</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>9</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3576491 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2019   \n",
       "1                00         00000     0000000    NaN      2019   \n",
       "2                00         00000     0000000    NaN      2019   \n",
       "3                00         00000     0000000    NaN      2019   \n",
       "4                00         00000     0000000    NaN      2019   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3576486          58         72769     5838305    NaN      2019   \n",
       "3576487          58         72769     5838305    NaN      2019   \n",
       "3576488          58         72769     5838305    NaN      2019   \n",
       "3576489          58         72769     5838305    NaN      2019   \n",
       "3576490          58         72769     5838305    NaN      2019   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    3165580   \n",
       "1                      3         B                    1616938   \n",
       "2                      4         B                    1548642   \n",
       "3                      6         B                    2651676   \n",
       "4                      7         B                     132465   \n",
       "...                  ...       ...                        ...   \n",
       "3576486              222         B                          *   \n",
       "3576487              223         B                          *   \n",
       "3576488              224         B                         19   \n",
       "3576489              226         B                         54   \n",
       "3576490              227         B                          9   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        3162910     3       1            453153   \n",
       "1                        1615465     3       1            231186   \n",
       "2                        1547445     3       1            221967   \n",
       "3                        2649540     3       1            352884   \n",
       "4                         132405     3       1             16409   \n",
       "...                          ...   ...     ...               ...   \n",
       "3576486                        *    13       2                 *   \n",
       "3576487                        *    13       2                 *   \n",
       "3576488                       19    13       2                19   \n",
       "3576489                       54    13       2                57   \n",
       "3576490                        9    13       2                 9   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                442858           2425.2            26.35       22.19   \n",
       "1                225530           2417.0            23.86       21.16   \n",
       "2                217328           2433.8            28.93       23.27   \n",
       "3                346840           2442.5            32.09       24.76   \n",
       "4                 16255           2491.5            54.47       23.98   \n",
       "...                 ...              ...              ...         ...   \n",
       "3576486               *              NaN                *           *   \n",
       "3576487               *              NaN                *           *   \n",
       "3576488              19              NaN             0.00       15.79   \n",
       "3576489              54              NaN             7.41       24.07   \n",
       "3576490               9              NaN                *           *   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   48.54              23.42           28.04   \n",
       "1                   45.02              23.54           31.44   \n",
       "2                   52.20              23.29           24.51   \n",
       "3                   56.85              22.39           20.76   \n",
       "4                   78.45              13.96            7.59   \n",
       "...                   ...                ...             ...   \n",
       "3576486                 *                  *               *   \n",
       "3576487                 *                  *               *   \n",
       "3576488             15.79              42.11           42.11   \n",
       "3576489             31.48              29.63           38.89   \n",
       "3576490                 *                  *               *   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     442660                26.64               45.35   \n",
       "1                     225427                24.12               44.79   \n",
       "2                     217233                29.26               45.93   \n",
       "3                     346684                32.26               46.50   \n",
       "4                      16252                52.57               38.73   \n",
       "...                      ...                  ...                 ...   \n",
       "3576486                    *                    *                   *   \n",
       "3576487                    *                    *                   *   \n",
       "3576488                   19                 5.26               36.84   \n",
       "3576489                   54                22.22               22.22   \n",
       "3576490                    9                    *                   *   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      28.01                20.83               49.66   \n",
       "1                      31.09                17.93               49.03   \n",
       "2                      24.81                23.85               50.32   \n",
       "3                      21.24                25.12               51.92   \n",
       "4                       8.70                42.74               47.23   \n",
       "...                      ...                  ...                 ...   \n",
       "3576486                    *                    *                   *   \n",
       "3576487                    *                    *                   *   \n",
       "3576488                57.89                 0.00               63.16   \n",
       "3576489                55.56                16.67               44.44   \n",
       "3576490                    *                    *                   *   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      29.50                21.89               61.25   \n",
       "1                      33.04                20.66               60.54   \n",
       "2                      25.83                23.17               61.99   \n",
       "3                      22.96                26.17               61.23   \n",
       "4                      10.03                42.25               53.14   \n",
       "...                      ...                  ...                 ...   \n",
       "3576486                    *                    *                   *   \n",
       "3576487                    *                    *                   *   \n",
       "3576488                36.84                 5.26               57.89   \n",
       "3576489                38.89                14.81               57.41   \n",
       "3576490                    *                    *                   *   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      16.86                24.14               48.16   \n",
       "1                      18.81                22.66               46.69   \n",
       "2                      14.84                25.68               49.69   \n",
       "3                      12.60                28.93               49.45   \n",
       "4                       4.60                47.75               43.21   \n",
       "...                      ...                  ...                 ...   \n",
       "3576486                    *                    *                   *   \n",
       "3576487                    *                    *                   *   \n",
       "3576488                36.84                 0.00                0.00   \n",
       "3576489                27.78                 0.00                0.00   \n",
       "3576490                    *                    *                   *   \n",
       "\n",
       "        area_4_pct_below_std type_id  \n",
       "0                      27.70       4  \n",
       "1                      30.65       4  \n",
       "2                      24.63       4  \n",
       "3                      21.62       4  \n",
       "4                       9.04       4  \n",
       "...                      ...     ...  \n",
       "3576486                    *       7  \n",
       "3576487                    *       7  \n",
       "3576488                 0.00       7  \n",
       "3576489                 0.00       7  \n",
       "3576490                    *       7  \n",
       "\n",
       "[3576491 rows x 33 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sb_ca2019 = pd.merge(sb_ca2019, df_type_id_final, on = ['county_code','district_code', 'school_code'], how='left')\n",
    "sb_ca2019"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3180571</td>\n",
       "      <td>3177420</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>445017</td>\n",
       "      <td>434468</td>\n",
       "      <td>2424.0</td>\n",
       "      <td>26.13</td>\n",
       "      <td>22.09</td>\n",
       "      <td>48.22</td>\n",
       "      <td>23.49</td>\n",
       "      <td>28.29</td>\n",
       "      <td>434207</td>\n",
       "      <td>25.32</td>\n",
       "      <td>44.02</td>\n",
       "      <td>30.66</td>\n",
       "      <td>23.84</td>\n",
       "      <td>43.30</td>\n",
       "      <td>32.85</td>\n",
       "      <td>20.89</td>\n",
       "      <td>61.25</td>\n",
       "      <td>17.86</td>\n",
       "      <td>27.60</td>\n",
       "      <td>47.71</td>\n",
       "      <td>24.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3187408</td>\n",
       "      <td>3184720</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>445018</td>\n",
       "      <td>436464</td>\n",
       "      <td>2430.9</td>\n",
       "      <td>21.07</td>\n",
       "      <td>27.82</td>\n",
       "      <td>48.89</td>\n",
       "      <td>23.56</td>\n",
       "      <td>27.55</td>\n",
       "      <td>436215</td>\n",
       "      <td>33.59</td>\n",
       "      <td>33.20</td>\n",
       "      <td>33.21</td>\n",
       "      <td>26.72</td>\n",
       "      <td>42.30</td>\n",
       "      <td>30.98</td>\n",
       "      <td>28.80</td>\n",
       "      <td>46.31</td>\n",
       "      <td>24.89</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3187408</td>\n",
       "      <td>3184720</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>463838</td>\n",
       "      <td>455619</td>\n",
       "      <td>2467.7</td>\n",
       "      <td>18.46</td>\n",
       "      <td>24.45</td>\n",
       "      <td>42.91</td>\n",
       "      <td>30.81</td>\n",
       "      <td>26.27</td>\n",
       "      <td>455345</td>\n",
       "      <td>29.03</td>\n",
       "      <td>31.02</td>\n",
       "      <td>39.95</td>\n",
       "      <td>21.65</td>\n",
       "      <td>44.73</td>\n",
       "      <td>33.62</td>\n",
       "      <td>24.03</td>\n",
       "      <td>43.78</td>\n",
       "      <td>32.20</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3180571</td>\n",
       "      <td>3177420</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>463838</td>\n",
       "      <td>453771</td>\n",
       "      <td>2463.7</td>\n",
       "      <td>26.31</td>\n",
       "      <td>22.36</td>\n",
       "      <td>48.67</td>\n",
       "      <td>19.25</td>\n",
       "      <td>32.08</td>\n",
       "      <td>453491</td>\n",
       "      <td>24.77</td>\n",
       "      <td>46.77</td>\n",
       "      <td>28.46</td>\n",
       "      <td>24.18</td>\n",
       "      <td>44.21</td>\n",
       "      <td>31.61</td>\n",
       "      <td>19.30</td>\n",
       "      <td>63.22</td>\n",
       "      <td>17.48</td>\n",
       "      <td>25.86</td>\n",
       "      <td>48.83</td>\n",
       "      <td>25.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3180571</td>\n",
       "      <td>3177420</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>469247</td>\n",
       "      <td>459434</td>\n",
       "      <td>2496.3</td>\n",
       "      <td>21.80</td>\n",
       "      <td>27.63</td>\n",
       "      <td>49.43</td>\n",
       "      <td>19.99</td>\n",
       "      <td>30.58</td>\n",
       "      <td>459209</td>\n",
       "      <td>24.22</td>\n",
       "      <td>45.02</td>\n",
       "      <td>30.76</td>\n",
       "      <td>29.20</td>\n",
       "      <td>41.42</td>\n",
       "      <td>29.38</td>\n",
       "      <td>16.52</td>\n",
       "      <td>59.74</td>\n",
       "      <td>23.73</td>\n",
       "      <td>28.29</td>\n",
       "      <td>44.32</td>\n",
       "      <td>27.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269725</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>226</td>\n",
       "      <td>B</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>53</td>\n",
       "      <td>52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13.46</td>\n",
       "      <td>19.23</td>\n",
       "      <td>32.69</td>\n",
       "      <td>32.69</td>\n",
       "      <td>34.62</td>\n",
       "      <td>52</td>\n",
       "      <td>21.15</td>\n",
       "      <td>44.23</td>\n",
       "      <td>34.62</td>\n",
       "      <td>21.15</td>\n",
       "      <td>44.23</td>\n",
       "      <td>34.62</td>\n",
       "      <td>17.31</td>\n",
       "      <td>50.00</td>\n",
       "      <td>32.69</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269726</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269727</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269728</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269729</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3269730 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2018   \n",
       "1                00         00000     0000000    NaN      2018   \n",
       "2                00         00000     0000000    NaN      2018   \n",
       "3                00         00000     0000000    NaN      2018   \n",
       "4                00         00000     0000000    NaN      2018   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3269725          58         72769     5838305    NaN      2018   \n",
       "3269726          58         72769     5838305    NaN      2018   \n",
       "3269727          58         72769     5838305    NaN      2018   \n",
       "3269728          58         72769     5838305    NaN      2018   \n",
       "3269729          58         72769     5838305    NaN      2018   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    3180571   \n",
       "1                      1         B                    3187408   \n",
       "2                      1         B                    3187408   \n",
       "3                      1         B                    3180571   \n",
       "4                      1         B                    3180571   \n",
       "...                  ...       ...                        ...   \n",
       "3269725              226         B                         52   \n",
       "3269726              227         B                          *   \n",
       "3269727              227         B                          *   \n",
       "3269728              227         B                          *   \n",
       "3269729              227         B                          *   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        3177420     3       1            445017   \n",
       "1                        3184720     3       2            445018   \n",
       "2                        3184720     4       2            463838   \n",
       "3                        3177420     4       1            463838   \n",
       "4                        3177420     5       1            469247   \n",
       "...                          ...   ...     ...               ...   \n",
       "3269725                       52    13       2                53   \n",
       "3269726                        *    11       1                 *   \n",
       "3269727                        *    11       2                 *   \n",
       "3269728                        *    13       2                 *   \n",
       "3269729                        *    13       1                 *   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                434468           2424.0            26.13       22.09   \n",
       "1                436464           2430.9            21.07       27.82   \n",
       "2                455619           2467.7            18.46       24.45   \n",
       "3                453771           2463.7            26.31       22.36   \n",
       "4                459434           2496.3            21.80       27.63   \n",
       "...                 ...              ...              ...         ...   \n",
       "3269725              52              NaN            13.46       19.23   \n",
       "3269726               *                *                *           *   \n",
       "3269727               *                *                *           *   \n",
       "3269728               *              NaN                *           *   \n",
       "3269729               *              NaN                *           *   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   48.22              23.49           28.29   \n",
       "1                   48.89              23.56           27.55   \n",
       "2                   42.91              30.81           26.27   \n",
       "3                   48.67              19.25           32.08   \n",
       "4                   49.43              19.99           30.58   \n",
       "...                   ...                ...             ...   \n",
       "3269725             32.69              32.69           34.62   \n",
       "3269726                 *                  *               *   \n",
       "3269727                 *                  *               *   \n",
       "3269728                 *                  *               *   \n",
       "3269729                 *                  *               *   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     434207                25.32               44.02   \n",
       "1                     436215                33.59               33.20   \n",
       "2                     455345                29.03               31.02   \n",
       "3                     453491                24.77               46.77   \n",
       "4                     459209                24.22               45.02   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                   52                21.15               44.23   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      30.66                23.84               43.30   \n",
       "1                      33.21                26.72               42.30   \n",
       "2                      39.95                21.65               44.73   \n",
       "3                      28.46                24.18               44.21   \n",
       "4                      30.76                29.20               41.42   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                34.62                21.15               44.23   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      32.85                20.89               61.25   \n",
       "1                      30.98                28.80               46.31   \n",
       "2                      33.62                24.03               43.78   \n",
       "3                      31.61                19.30               63.22   \n",
       "4                      29.38                16.52               59.74   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                34.62                17.31               50.00   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      17.86                27.60               47.71   \n",
       "1                      24.89                 0.00                0.00   \n",
       "2                      32.20                 0.00                0.00   \n",
       "3                      17.48                25.86               48.83   \n",
       "4                      23.73                28.29               44.32   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                32.69                 0.00                0.00   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_4_pct_below_std  \n",
       "0                      24.68  \n",
       "1                       0.00  \n",
       "2                       0.00  \n",
       "3                      25.31  \n",
       "4                      27.39  \n",
       "...                      ...  \n",
       "3269725                 0.00  \n",
       "3269726                    *  \n",
       "3269727                    *  \n",
       "3269728                    *  \n",
       "3269729                    *  \n",
       "\n",
       "[3269730 rows x 32 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sb_ca2018 = pd.read_csv('caaspp_csv_files_exported/sb_ca2018_all_csv_v3.csv', encoding='ISO-8859-1',dtype=str)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "sb_ca2018.columns = [\n",
    "    'county_code',\n",
    "    'district_code',\n",
    "    'school_code',\n",
    "    'filler',\n",
    "    'test_year',\n",
    "    'student_group_id',\n",
    "    'test_type',\n",
    "    'total_tested_reporting_lvl',\n",
    "    'total_tested_with_scores',\n",
    "    'grade',\n",
    "    'test_id',\n",
    "    'students_enrolled',\n",
    "    'students_tested',\n",
    "    'mean_scale_score',\n",
    "    'pct_std_exceeded',\n",
    "    'pct_std_met',\n",
    "    'pct_std_met_above',\n",
    "    'pct_std_nearly_met',\n",
    "    'pct_std_not_met',\n",
    "    'students_with_scores',\n",
    "    'area_1_pct_above_std',\n",
    "    'area_1_pct_near_std',\n",
    "    'area_1_pct_below_std',\n",
    "    'area_2_pct_above_std',\n",
    "    'area_2_pct_near_std',\n",
    "    'area_2_pct_below_std',\n",
    "    'area_3_pct_above_std',\n",
    "    'area_3_pct_near_std',\n",
    "    'area_3_pct_below_std',\n",
    "    'area_4_pct_above_std',\n",
    "    'area_4_pct_near_std',\n",
    "    'area_4_pct_below_std'\n",
    "]\n",
    "\n",
    "sb_ca2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3180571</td>\n",
       "      <td>3177420</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>445017</td>\n",
       "      <td>434468</td>\n",
       "      <td>2424.0</td>\n",
       "      <td>26.13</td>\n",
       "      <td>22.09</td>\n",
       "      <td>48.22</td>\n",
       "      <td>23.49</td>\n",
       "      <td>28.29</td>\n",
       "      <td>434207</td>\n",
       "      <td>25.32</td>\n",
       "      <td>44.02</td>\n",
       "      <td>30.66</td>\n",
       "      <td>23.84</td>\n",
       "      <td>43.30</td>\n",
       "      <td>32.85</td>\n",
       "      <td>20.89</td>\n",
       "      <td>61.25</td>\n",
       "      <td>17.86</td>\n",
       "      <td>27.60</td>\n",
       "      <td>47.71</td>\n",
       "      <td>24.68</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3187408</td>\n",
       "      <td>3184720</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>445018</td>\n",
       "      <td>436464</td>\n",
       "      <td>2430.9</td>\n",
       "      <td>21.07</td>\n",
       "      <td>27.82</td>\n",
       "      <td>48.89</td>\n",
       "      <td>23.56</td>\n",
       "      <td>27.55</td>\n",
       "      <td>436215</td>\n",
       "      <td>33.59</td>\n",
       "      <td>33.20</td>\n",
       "      <td>33.21</td>\n",
       "      <td>26.72</td>\n",
       "      <td>42.30</td>\n",
       "      <td>30.98</td>\n",
       "      <td>28.80</td>\n",
       "      <td>46.31</td>\n",
       "      <td>24.89</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3187408</td>\n",
       "      <td>3184720</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>463838</td>\n",
       "      <td>455619</td>\n",
       "      <td>2467.7</td>\n",
       "      <td>18.46</td>\n",
       "      <td>24.45</td>\n",
       "      <td>42.91</td>\n",
       "      <td>30.81</td>\n",
       "      <td>26.27</td>\n",
       "      <td>455345</td>\n",
       "      <td>29.03</td>\n",
       "      <td>31.02</td>\n",
       "      <td>39.95</td>\n",
       "      <td>21.65</td>\n",
       "      <td>44.73</td>\n",
       "      <td>33.62</td>\n",
       "      <td>24.03</td>\n",
       "      <td>43.78</td>\n",
       "      <td>32.20</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3180571</td>\n",
       "      <td>3177420</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>463838</td>\n",
       "      <td>453771</td>\n",
       "      <td>2463.7</td>\n",
       "      <td>26.31</td>\n",
       "      <td>22.36</td>\n",
       "      <td>48.67</td>\n",
       "      <td>19.25</td>\n",
       "      <td>32.08</td>\n",
       "      <td>453491</td>\n",
       "      <td>24.77</td>\n",
       "      <td>46.77</td>\n",
       "      <td>28.46</td>\n",
       "      <td>24.18</td>\n",
       "      <td>44.21</td>\n",
       "      <td>31.61</td>\n",
       "      <td>19.30</td>\n",
       "      <td>63.22</td>\n",
       "      <td>17.48</td>\n",
       "      <td>25.86</td>\n",
       "      <td>48.83</td>\n",
       "      <td>25.31</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>3180571</td>\n",
       "      <td>3177420</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>469247</td>\n",
       "      <td>459434</td>\n",
       "      <td>2496.3</td>\n",
       "      <td>21.80</td>\n",
       "      <td>27.63</td>\n",
       "      <td>49.43</td>\n",
       "      <td>19.99</td>\n",
       "      <td>30.58</td>\n",
       "      <td>459209</td>\n",
       "      <td>24.22</td>\n",
       "      <td>45.02</td>\n",
       "      <td>30.76</td>\n",
       "      <td>29.20</td>\n",
       "      <td>41.42</td>\n",
       "      <td>29.38</td>\n",
       "      <td>16.52</td>\n",
       "      <td>59.74</td>\n",
       "      <td>23.73</td>\n",
       "      <td>28.29</td>\n",
       "      <td>44.32</td>\n",
       "      <td>27.39</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269725</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>226</td>\n",
       "      <td>B</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>53</td>\n",
       "      <td>52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13.46</td>\n",
       "      <td>19.23</td>\n",
       "      <td>32.69</td>\n",
       "      <td>32.69</td>\n",
       "      <td>34.62</td>\n",
       "      <td>52</td>\n",
       "      <td>21.15</td>\n",
       "      <td>44.23</td>\n",
       "      <td>34.62</td>\n",
       "      <td>21.15</td>\n",
       "      <td>44.23</td>\n",
       "      <td>34.62</td>\n",
       "      <td>17.31</td>\n",
       "      <td>50.00</td>\n",
       "      <td>32.69</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269726</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269727</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269728</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269729</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3269730 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2018   \n",
       "1                00         00000     0000000    NaN      2018   \n",
       "2                00         00000     0000000    NaN      2018   \n",
       "3                00         00000     0000000    NaN      2018   \n",
       "4                00         00000     0000000    NaN      2018   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3269725          58         72769     5838305    NaN      2018   \n",
       "3269726          58         72769     5838305    NaN      2018   \n",
       "3269727          58         72769     5838305    NaN      2018   \n",
       "3269728          58         72769     5838305    NaN      2018   \n",
       "3269729          58         72769     5838305    NaN      2018   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    3180571   \n",
       "1                      1         B                    3187408   \n",
       "2                      1         B                    3187408   \n",
       "3                      1         B                    3180571   \n",
       "4                      1         B                    3180571   \n",
       "...                  ...       ...                        ...   \n",
       "3269725              226         B                         52   \n",
       "3269726              227         B                          *   \n",
       "3269727              227         B                          *   \n",
       "3269728              227         B                          *   \n",
       "3269729              227         B                          *   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        3177420     3       1            445017   \n",
       "1                        3184720     3       2            445018   \n",
       "2                        3184720     4       2            463838   \n",
       "3                        3177420     4       1            463838   \n",
       "4                        3177420     5       1            469247   \n",
       "...                          ...   ...     ...               ...   \n",
       "3269725                       52    13       2                53   \n",
       "3269726                        *    11       1                 *   \n",
       "3269727                        *    11       2                 *   \n",
       "3269728                        *    13       2                 *   \n",
       "3269729                        *    13       1                 *   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                434468           2424.0            26.13       22.09   \n",
       "1                436464           2430.9            21.07       27.82   \n",
       "2                455619           2467.7            18.46       24.45   \n",
       "3                453771           2463.7            26.31       22.36   \n",
       "4                459434           2496.3            21.80       27.63   \n",
       "...                 ...              ...              ...         ...   \n",
       "3269725              52              NaN            13.46       19.23   \n",
       "3269726               *                *                *           *   \n",
       "3269727               *                *                *           *   \n",
       "3269728               *              NaN                *           *   \n",
       "3269729               *              NaN                *           *   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   48.22              23.49           28.29   \n",
       "1                   48.89              23.56           27.55   \n",
       "2                   42.91              30.81           26.27   \n",
       "3                   48.67              19.25           32.08   \n",
       "4                   49.43              19.99           30.58   \n",
       "...                   ...                ...             ...   \n",
       "3269725             32.69              32.69           34.62   \n",
       "3269726                 *                  *               *   \n",
       "3269727                 *                  *               *   \n",
       "3269728                 *                  *               *   \n",
       "3269729                 *                  *               *   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     434207                25.32               44.02   \n",
       "1                     436215                33.59               33.20   \n",
       "2                     455345                29.03               31.02   \n",
       "3                     453491                24.77               46.77   \n",
       "4                     459209                24.22               45.02   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                   52                21.15               44.23   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      30.66                23.84               43.30   \n",
       "1                      33.21                26.72               42.30   \n",
       "2                      39.95                21.65               44.73   \n",
       "3                      28.46                24.18               44.21   \n",
       "4                      30.76                29.20               41.42   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                34.62                21.15               44.23   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      32.85                20.89               61.25   \n",
       "1                      30.98                28.80               46.31   \n",
       "2                      33.62                24.03               43.78   \n",
       "3                      31.61                19.30               63.22   \n",
       "4                      29.38                16.52               59.74   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                34.62                17.31               50.00   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      17.86                27.60               47.71   \n",
       "1                      24.89                 0.00                0.00   \n",
       "2                      32.20                 0.00                0.00   \n",
       "3                      17.48                25.86               48.83   \n",
       "4                      23.73                28.29               44.32   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                32.69                 0.00                0.00   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_4_pct_below_std type_id  \n",
       "0                      24.68       4  \n",
       "1                       0.00       4  \n",
       "2                       0.00       4  \n",
       "3                      25.31       4  \n",
       "4                      27.39       4  \n",
       "...                      ...     ...  \n",
       "3269725                 0.00       7  \n",
       "3269726                    *       7  \n",
       "3269727                    *       7  \n",
       "3269728                    *       7  \n",
       "3269729                    *       7  \n",
       "\n",
       "[3269730 rows x 33 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sb_ca2018 = pd.merge(sb_ca2018, df_type_id_final, on = ['county_code','district_code', 'school_code'], how='left')\n",
    "sb_ca2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2960990</td>\n",
       "      <td>2957491</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>420238</td>\n",
       "      <td>410559</td>\n",
       "      <td>2409.8</td>\n",
       "      <td>23.40</td>\n",
       "      <td>19.56</td>\n",
       "      <td>42.96</td>\n",
       "      <td>22.40</td>\n",
       "      <td>34.64</td>\n",
       "      <td>410290</td>\n",
       "      <td>17.47</td>\n",
       "      <td>57.69</td>\n",
       "      <td>24.84</td>\n",
       "      <td>16.18</td>\n",
       "      <td>50.24</td>\n",
       "      <td>33.58</td>\n",
       "      <td>11.75</td>\n",
       "      <td>70.96</td>\n",
       "      <td>17.29</td>\n",
       "      <td>16.07</td>\n",
       "      <td>60.00</td>\n",
       "      <td>23.94</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1512977</td>\n",
       "      <td>1511175</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>214436</td>\n",
       "      <td>209138</td>\n",
       "      <td>2402.4</td>\n",
       "      <td>21.27</td>\n",
       "      <td>18.77</td>\n",
       "      <td>40.04</td>\n",
       "      <td>22.45</td>\n",
       "      <td>37.51</td>\n",
       "      <td>208998</td>\n",
       "      <td>15.88</td>\n",
       "      <td>57.23</td>\n",
       "      <td>26.89</td>\n",
       "      <td>13.96</td>\n",
       "      <td>49.35</td>\n",
       "      <td>36.69</td>\n",
       "      <td>11.47</td>\n",
       "      <td>70.55</td>\n",
       "      <td>17.99</td>\n",
       "      <td>15.20</td>\n",
       "      <td>59.33</td>\n",
       "      <td>25.47</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1445927</td>\n",
       "      <td>1444241</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>205715</td>\n",
       "      <td>201341</td>\n",
       "      <td>2417.5</td>\n",
       "      <td>25.60</td>\n",
       "      <td>20.39</td>\n",
       "      <td>45.99</td>\n",
       "      <td>22.34</td>\n",
       "      <td>31.67</td>\n",
       "      <td>201212</td>\n",
       "      <td>19.12</td>\n",
       "      <td>58.17</td>\n",
       "      <td>22.71</td>\n",
       "      <td>18.48</td>\n",
       "      <td>51.16</td>\n",
       "      <td>30.36</td>\n",
       "      <td>12.04</td>\n",
       "      <td>71.39</td>\n",
       "      <td>16.57</td>\n",
       "      <td>16.96</td>\n",
       "      <td>60.69</td>\n",
       "      <td>22.35</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>6</td>\n",
       "      <td>B</td>\n",
       "      <td>2459254</td>\n",
       "      <td>2456394</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>321484</td>\n",
       "      <td>316733</td>\n",
       "      <td>2428.5</td>\n",
       "      <td>28.93</td>\n",
       "      <td>21.99</td>\n",
       "      <td>50.92</td>\n",
       "      <td>21.81</td>\n",
       "      <td>27.27</td>\n",
       "      <td>316516</td>\n",
       "      <td>21.56</td>\n",
       "      <td>59.05</td>\n",
       "      <td>19.39</td>\n",
       "      <td>19.88</td>\n",
       "      <td>52.83</td>\n",
       "      <td>27.29</td>\n",
       "      <td>14.03</td>\n",
       "      <td>71.81</td>\n",
       "      <td>14.16</td>\n",
       "      <td>19.54</td>\n",
       "      <td>61.39</td>\n",
       "      <td>19.08</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>7</td>\n",
       "      <td>B</td>\n",
       "      <td>144219</td>\n",
       "      <td>144145</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>27927</td>\n",
       "      <td>27764</td>\n",
       "      <td>2462.9</td>\n",
       "      <td>40.62</td>\n",
       "      <td>23.73</td>\n",
       "      <td>64.35</td>\n",
       "      <td>19.87</td>\n",
       "      <td>15.78</td>\n",
       "      <td>27760</td>\n",
       "      <td>30.24</td>\n",
       "      <td>58.31</td>\n",
       "      <td>11.45</td>\n",
       "      <td>28.50</td>\n",
       "      <td>54.52</td>\n",
       "      <td>16.98</td>\n",
       "      <td>19.43</td>\n",
       "      <td>71.38</td>\n",
       "      <td>9.19</td>\n",
       "      <td>28.13</td>\n",
       "      <td>60.23</td>\n",
       "      <td>11.65</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269725</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>226</td>\n",
       "      <td>B</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>53</td>\n",
       "      <td>52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13.46</td>\n",
       "      <td>19.23</td>\n",
       "      <td>32.69</td>\n",
       "      <td>32.69</td>\n",
       "      <td>34.62</td>\n",
       "      <td>52</td>\n",
       "      <td>21.15</td>\n",
       "      <td>44.23</td>\n",
       "      <td>34.62</td>\n",
       "      <td>21.15</td>\n",
       "      <td>44.23</td>\n",
       "      <td>34.62</td>\n",
       "      <td>17.31</td>\n",
       "      <td>50.00</td>\n",
       "      <td>32.69</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269726</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269727</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269728</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269729</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>227</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18341307 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2023   \n",
       "1                00         00000     0000000    NaN      2023   \n",
       "2                00         00000     0000000    NaN      2023   \n",
       "3                00         00000     0000000    NaN      2023   \n",
       "4                00         00000     0000000    NaN      2023   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3269725          58         72769     5838305    NaN      2018   \n",
       "3269726          58         72769     5838305    NaN      2018   \n",
       "3269727          58         72769     5838305    NaN      2018   \n",
       "3269728          58         72769     5838305    NaN      2018   \n",
       "3269729          58         72769     5838305    NaN      2018   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    2960990   \n",
       "1                      3         B                    1512977   \n",
       "2                      4         B                    1445927   \n",
       "3                      6         B                    2459254   \n",
       "4                      7         B                     144219   \n",
       "...                  ...       ...                        ...   \n",
       "3269725              226         B                         52   \n",
       "3269726              227         B                          *   \n",
       "3269727              227         B                          *   \n",
       "3269728              227         B                          *   \n",
       "3269729              227         B                          *   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        2957491     3       1            420238   \n",
       "1                        1511175     3       1            214436   \n",
       "2                        1444241     3       1            205715   \n",
       "3                        2456394     3       1            321484   \n",
       "4                         144145     3       1             27927   \n",
       "...                          ...   ...     ...               ...   \n",
       "3269725                       52    13       2                53   \n",
       "3269726                        *    11       1                 *   \n",
       "3269727                        *    11       2                 *   \n",
       "3269728                        *    13       2                 *   \n",
       "3269729                        *    13       1                 *   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                410559           2409.8            23.40       19.56   \n",
       "1                209138           2402.4            21.27       18.77   \n",
       "2                201341           2417.5            25.60       20.39   \n",
       "3                316733           2428.5            28.93       21.99   \n",
       "4                 27764           2462.9            40.62       23.73   \n",
       "...                 ...              ...              ...         ...   \n",
       "3269725              52              NaN            13.46       19.23   \n",
       "3269726               *                *                *           *   \n",
       "3269727               *                *                *           *   \n",
       "3269728               *              NaN                *           *   \n",
       "3269729               *              NaN                *           *   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   42.96              22.40           34.64   \n",
       "1                   40.04              22.45           37.51   \n",
       "2                   45.99              22.34           31.67   \n",
       "3                   50.92              21.81           27.27   \n",
       "4                   64.35              19.87           15.78   \n",
       "...                   ...                ...             ...   \n",
       "3269725             32.69              32.69           34.62   \n",
       "3269726                 *                  *               *   \n",
       "3269727                 *                  *               *   \n",
       "3269728                 *                  *               *   \n",
       "3269729                 *                  *               *   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     410290                17.47               57.69   \n",
       "1                     208998                15.88               57.23   \n",
       "2                     201212                19.12               58.17   \n",
       "3                     316516                21.56               59.05   \n",
       "4                      27760                30.24               58.31   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                   52                21.15               44.23   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      24.84                16.18               50.24   \n",
       "1                      26.89                13.96               49.35   \n",
       "2                      22.71                18.48               51.16   \n",
       "3                      19.39                19.88               52.83   \n",
       "4                      11.45                28.50               54.52   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                34.62                21.15               44.23   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      33.58                11.75               70.96   \n",
       "1                      36.69                11.47               70.55   \n",
       "2                      30.36                12.04               71.39   \n",
       "3                      27.29                14.03               71.81   \n",
       "4                      16.98                19.43               71.38   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                34.62                17.31               50.00   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      17.29                16.07               60.00   \n",
       "1                      17.99                15.20               59.33   \n",
       "2                      16.57                16.96               60.69   \n",
       "3                      14.16                19.54               61.39   \n",
       "4                       9.19                28.13               60.23   \n",
       "...                      ...                  ...                 ...   \n",
       "3269725                32.69                 0.00                0.00   \n",
       "3269726                    *                    *                   *   \n",
       "3269727                    *                    *                   *   \n",
       "3269728                    *                    *                   *   \n",
       "3269729                    *                    *                   *   \n",
       "\n",
       "        area_4_pct_below_std type_id  \n",
       "0                      23.94       4  \n",
       "1                      25.47       4  \n",
       "2                      22.35       4  \n",
       "3                      19.08       4  \n",
       "4                      11.65       4  \n",
       "...                      ...     ...  \n",
       "3269725                 0.00       7  \n",
       "3269726                    *       7  \n",
       "3269727                    *       7  \n",
       "3269728                    *       7  \n",
       "3269729                    *       7  \n",
       "\n",
       "[18341307 rows x 33 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_caaspp_df = pd.concat([sb_ca2023, sb_ca2022, sb_ca2021, sb_ca2019, sb_ca2018])\n",
    "final_caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of string values you want to filter by\n",
    "student_groups_to_include = [\n",
    "   '1', '3', '4', '8', '75', '76', '74', '77', '78', '79', '80', '144', '160', '52', '28', '31', '128', '240'\n",
    "]\n",
    "\n",
    "# Use .loc with your condition to filter 'final_caaspp_df' in place\n",
    "# This will keep only the rows where 'student_group_id' matches your criteria\n",
    "final_caaspp_df = final_caaspp_df.loc[final_caaspp_df['student_group_id'].isin(student_groups_to_include)]\n",
    "\n",
    "# At this point, 'final_caaspp_df' only contains rows that match the condition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/6t/n3rqwc2x06ngf2r8521g1nmc0000gn/T/ipykernel_19588/407944674.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  final_caaspp_df['is_wcc_entity'] = final_caaspp_df['school_code'].isin(wcc_schools_set)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2960990</td>\n",
       "      <td>2957491</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>420238</td>\n",
       "      <td>410559</td>\n",
       "      <td>2409.8</td>\n",
       "      <td>23.40</td>\n",
       "      <td>19.56</td>\n",
       "      <td>42.96</td>\n",
       "      <td>22.40</td>\n",
       "      <td>34.64</td>\n",
       "      <td>410290</td>\n",
       "      <td>17.47</td>\n",
       "      <td>57.69</td>\n",
       "      <td>24.84</td>\n",
       "      <td>16.18</td>\n",
       "      <td>50.24</td>\n",
       "      <td>33.58</td>\n",
       "      <td>11.75</td>\n",
       "      <td>70.96</td>\n",
       "      <td>17.29</td>\n",
       "      <td>16.07</td>\n",
       "      <td>60.00</td>\n",
       "      <td>23.94</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1512977</td>\n",
       "      <td>1511175</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>214436</td>\n",
       "      <td>209138</td>\n",
       "      <td>2402.4</td>\n",
       "      <td>21.27</td>\n",
       "      <td>18.77</td>\n",
       "      <td>40.04</td>\n",
       "      <td>22.45</td>\n",
       "      <td>37.51</td>\n",
       "      <td>208998</td>\n",
       "      <td>15.88</td>\n",
       "      <td>57.23</td>\n",
       "      <td>26.89</td>\n",
       "      <td>13.96</td>\n",
       "      <td>49.35</td>\n",
       "      <td>36.69</td>\n",
       "      <td>11.47</td>\n",
       "      <td>70.55</td>\n",
       "      <td>17.99</td>\n",
       "      <td>15.20</td>\n",
       "      <td>59.33</td>\n",
       "      <td>25.47</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1445927</td>\n",
       "      <td>1444241</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>205715</td>\n",
       "      <td>201341</td>\n",
       "      <td>2417.5</td>\n",
       "      <td>25.60</td>\n",
       "      <td>20.39</td>\n",
       "      <td>45.99</td>\n",
       "      <td>22.34</td>\n",
       "      <td>31.67</td>\n",
       "      <td>201212</td>\n",
       "      <td>19.12</td>\n",
       "      <td>58.17</td>\n",
       "      <td>22.71</td>\n",
       "      <td>18.48</td>\n",
       "      <td>51.16</td>\n",
       "      <td>30.36</td>\n",
       "      <td>12.04</td>\n",
       "      <td>71.39</td>\n",
       "      <td>16.57</td>\n",
       "      <td>16.96</td>\n",
       "      <td>60.69</td>\n",
       "      <td>22.35</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>8</td>\n",
       "      <td>B</td>\n",
       "      <td>557974</td>\n",
       "      <td>557303</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>24566</td>\n",
       "      <td>24451</td>\n",
       "      <td>2474.8</td>\n",
       "      <td>43.01</td>\n",
       "      <td>29.94</td>\n",
       "      <td>72.96</td>\n",
       "      <td>19.26</td>\n",
       "      <td>7.78</td>\n",
       "      <td>24435</td>\n",
       "      <td>29.83</td>\n",
       "      <td>63.91</td>\n",
       "      <td>6.26</td>\n",
       "      <td>29.77</td>\n",
       "      <td>60.20</td>\n",
       "      <td>10.02</td>\n",
       "      <td>18.18</td>\n",
       "      <td>74.83</td>\n",
       "      <td>6.98</td>\n",
       "      <td>28.87</td>\n",
       "      <td>64.16</td>\n",
       "      <td>6.97</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>28</td>\n",
       "      <td>B</td>\n",
       "      <td>22322</td>\n",
       "      <td>22290</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3265</td>\n",
       "      <td>3165</td>\n",
       "      <td>2344.5</td>\n",
       "      <td>7.30</td>\n",
       "      <td>10.30</td>\n",
       "      <td>17.60</td>\n",
       "      <td>21.04</td>\n",
       "      <td>61.36</td>\n",
       "      <td>3165</td>\n",
       "      <td>5.81</td>\n",
       "      <td>48.34</td>\n",
       "      <td>45.85</td>\n",
       "      <td>4.84</td>\n",
       "      <td>37.61</td>\n",
       "      <td>57.55</td>\n",
       "      <td>4.96</td>\n",
       "      <td>65.91</td>\n",
       "      <td>29.13</td>\n",
       "      <td>6.32</td>\n",
       "      <td>51.53</td>\n",
       "      <td>42.15</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269665</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269666</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>160</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269667</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>160</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269668</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>160</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269669</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>5838305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>160</td>\n",
       "      <td>B</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>NaN</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6345386 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2023   \n",
       "1                00         00000     0000000    NaN      2023   \n",
       "2                00         00000     0000000    NaN      2023   \n",
       "5                00         00000     0000000    NaN      2023   \n",
       "6                00         00000     0000000    NaN      2023   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3269665          58         72769     5838305    NaN      2018   \n",
       "3269666          58         72769     5838305    NaN      2018   \n",
       "3269667          58         72769     5838305    NaN      2018   \n",
       "3269668          58         72769     5838305    NaN      2018   \n",
       "3269669          58         72769     5838305    NaN      2018   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    2960990   \n",
       "1                      3         B                    1512977   \n",
       "2                      4         B                    1445927   \n",
       "5                      8         B                     557974   \n",
       "6                     28         B                      22322   \n",
       "...                  ...       ...                        ...   \n",
       "3269665              144         B                         20   \n",
       "3269666              160         B                          *   \n",
       "3269667              160         B                          *   \n",
       "3269668              160         B                          *   \n",
       "3269669              160         B                          *   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        2957491     3       1            420238   \n",
       "1                        1511175     3       1            214436   \n",
       "2                        1444241     3       1            205715   \n",
       "5                         557303     3       1             24566   \n",
       "6                          22290     3       1              3265   \n",
       "...                          ...   ...     ...               ...   \n",
       "3269665                       20    13       1                20   \n",
       "3269666                        *    11       1                 *   \n",
       "3269667                        *    11       2                 *   \n",
       "3269668                        *    13       2                 *   \n",
       "3269669                        *    13       1                 *   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                410559           2409.8            23.40       19.56   \n",
       "1                209138           2402.4            21.27       18.77   \n",
       "2                201341           2417.5            25.60       20.39   \n",
       "5                 24451           2474.8            43.01       29.94   \n",
       "6                  3165           2344.5             7.30       10.30   \n",
       "...                 ...              ...              ...         ...   \n",
       "3269665              20              NaN            30.00       20.00   \n",
       "3269666               *                *                *           *   \n",
       "3269667               *                *                *           *   \n",
       "3269668               *              NaN                *           *   \n",
       "3269669               *              NaN                *           *   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   42.96              22.40           34.64   \n",
       "1                   40.04              22.45           37.51   \n",
       "2                   45.99              22.34           31.67   \n",
       "5                   72.96              19.26            7.78   \n",
       "6                   17.60              21.04           61.36   \n",
       "...                   ...                ...             ...   \n",
       "3269665             50.00              25.00           25.00   \n",
       "3269666                 *                  *               *   \n",
       "3269667                 *                  *               *   \n",
       "3269668                 *                  *               *   \n",
       "3269669                 *                  *               *   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     410290                17.47               57.69   \n",
       "1                     208998                15.88               57.23   \n",
       "2                     201212                19.12               58.17   \n",
       "5                      24435                29.83               63.91   \n",
       "6                       3165                 5.81               48.34   \n",
       "...                      ...                  ...                 ...   \n",
       "3269665                   20                45.00               30.00   \n",
       "3269666                    *                    *                   *   \n",
       "3269667                    *                    *                   *   \n",
       "3269668                    *                    *                   *   \n",
       "3269669                    *                    *                   *   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      24.84                16.18               50.24   \n",
       "1                      26.89                13.96               49.35   \n",
       "2                      22.71                18.48               51.16   \n",
       "5                       6.26                29.77               60.20   \n",
       "6                      45.85                 4.84               37.61   \n",
       "...                      ...                  ...                 ...   \n",
       "3269665                25.00                35.00               30.00   \n",
       "3269666                    *                    *                   *   \n",
       "3269667                    *                    *                   *   \n",
       "3269668                    *                    *                   *   \n",
       "3269669                    *                    *                   *   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      33.58                11.75               70.96   \n",
       "1                      36.69                11.47               70.55   \n",
       "2                      30.36                12.04               71.39   \n",
       "5                      10.02                18.18               74.83   \n",
       "6                      57.55                 4.96               65.91   \n",
       "...                      ...                  ...                 ...   \n",
       "3269665                35.00                25.00               65.00   \n",
       "3269666                    *                    *                   *   \n",
       "3269667                    *                    *                   *   \n",
       "3269668                    *                    *                   *   \n",
       "3269669                    *                    *                   *   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      17.29                16.07               60.00   \n",
       "1                      17.99                15.20               59.33   \n",
       "2                      16.57                16.96               60.69   \n",
       "5                       6.98                28.87               64.16   \n",
       "6                      29.13                 6.32               51.53   \n",
       "...                      ...                  ...                 ...   \n",
       "3269665                10.00                30.00               35.00   \n",
       "3269666                    *                    *                   *   \n",
       "3269667                    *                    *                   *   \n",
       "3269668                    *                    *                   *   \n",
       "3269669                    *                    *                   *   \n",
       "\n",
       "        area_4_pct_below_std type_id  is_wcc_entity  \n",
       "0                      23.94       4          False  \n",
       "1                      25.47       4          False  \n",
       "2                      22.35       4          False  \n",
       "5                       6.97       4          False  \n",
       "6                      42.15       4          False  \n",
       "...                      ...     ...            ...  \n",
       "3269665                35.00       7          False  \n",
       "3269666                    *       7          False  \n",
       "3269667                    *       7          False  \n",
       "3269668                    *       7          False  \n",
       "3269669                    *       7          False  \n",
       "\n",
       "[6345386 rows x 34 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Step 1: Create a set of unique school codes from wcc_schools_df for efficient lookup\n",
    "wcc_schools_set = set(wcc_schools_df['School Code'])\n",
    "\n",
    "# Step 2: Check if each school code in final_caaspp_df is in the set of WCC schools\n",
    "# and directly assign the boolean series to the new 'is_wcc_entity' column.\n",
    "final_caaspp_df['is_wcc_entity'] = final_caaspp_df['school_code'].isin(wcc_schools_set)\n",
    "final_caaspp_df\n",
    "# final_caaspp_df now includes the 'is_wcc_entity' column indicating WCC schools\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['3', '4', '5', '6', '7', '8', '11', '13'], dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_caaspp_df['grade'].unique()\n",
    "# 3, 4, 5\n",
    "# 6, 7, 8 \n",
    "# 11 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>county_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>school_code</th>\n",
       "      <th>filler</th>\n",
       "      <th>test_year</th>\n",
       "      <th>student_group_id</th>\n",
       "      <th>test_type</th>\n",
       "      <th>total_tested_reporting_lvl</th>\n",
       "      <th>total_tested_with_scores</th>\n",
       "      <th>grade</th>\n",
       "      <th>test_id</th>\n",
       "      <th>students_enrolled</th>\n",
       "      <th>students_tested</th>\n",
       "      <th>mean_scale_score</th>\n",
       "      <th>pct_std_exceeded</th>\n",
       "      <th>pct_std_met</th>\n",
       "      <th>pct_std_met_above</th>\n",
       "      <th>pct_std_nearly_met</th>\n",
       "      <th>pct_std_not_met</th>\n",
       "      <th>students_with_scores</th>\n",
       "      <th>area_1_pct_above_std</th>\n",
       "      <th>area_1_pct_near_std</th>\n",
       "      <th>area_1_pct_below_std</th>\n",
       "      <th>area_2_pct_above_std</th>\n",
       "      <th>area_2_pct_near_std</th>\n",
       "      <th>area_2_pct_below_std</th>\n",
       "      <th>area_3_pct_above_std</th>\n",
       "      <th>area_3_pct_near_std</th>\n",
       "      <th>area_3_pct_below_std</th>\n",
       "      <th>area_4_pct_above_std</th>\n",
       "      <th>area_4_pct_near_std</th>\n",
       "      <th>area_4_pct_below_std</th>\n",
       "      <th>type_id</th>\n",
       "      <th>is_wcc_entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>2960990</td>\n",
       "      <td>2957491</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>420238</td>\n",
       "      <td>410559</td>\n",
       "      <td>2409.8</td>\n",
       "      <td>23.40</td>\n",
       "      <td>19.56</td>\n",
       "      <td>42.96</td>\n",
       "      <td>22.40</td>\n",
       "      <td>34.64</td>\n",
       "      <td>410290</td>\n",
       "      <td>17.47</td>\n",
       "      <td>57.69</td>\n",
       "      <td>24.84</td>\n",
       "      <td>16.18</td>\n",
       "      <td>50.24</td>\n",
       "      <td>33.58</td>\n",
       "      <td>11.75</td>\n",
       "      <td>70.96</td>\n",
       "      <td>17.29</td>\n",
       "      <td>16.07</td>\n",
       "      <td>60.00</td>\n",
       "      <td>23.94</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>3</td>\n",
       "      <td>B</td>\n",
       "      <td>1512977</td>\n",
       "      <td>1511175</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>214436</td>\n",
       "      <td>209138</td>\n",
       "      <td>2402.4</td>\n",
       "      <td>21.27</td>\n",
       "      <td>18.77</td>\n",
       "      <td>40.04</td>\n",
       "      <td>22.45</td>\n",
       "      <td>37.51</td>\n",
       "      <td>208998</td>\n",
       "      <td>15.88</td>\n",
       "      <td>57.23</td>\n",
       "      <td>26.89</td>\n",
       "      <td>13.96</td>\n",
       "      <td>49.35</td>\n",
       "      <td>36.69</td>\n",
       "      <td>11.47</td>\n",
       "      <td>70.55</td>\n",
       "      <td>17.99</td>\n",
       "      <td>15.20</td>\n",
       "      <td>59.33</td>\n",
       "      <td>25.47</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>1445927</td>\n",
       "      <td>1444241</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>205715</td>\n",
       "      <td>201341</td>\n",
       "      <td>2417.5</td>\n",
       "      <td>25.60</td>\n",
       "      <td>20.39</td>\n",
       "      <td>45.99</td>\n",
       "      <td>22.34</td>\n",
       "      <td>31.67</td>\n",
       "      <td>201212</td>\n",
       "      <td>19.12</td>\n",
       "      <td>58.17</td>\n",
       "      <td>22.71</td>\n",
       "      <td>18.48</td>\n",
       "      <td>51.16</td>\n",
       "      <td>30.36</td>\n",
       "      <td>12.04</td>\n",
       "      <td>71.39</td>\n",
       "      <td>16.57</td>\n",
       "      <td>16.96</td>\n",
       "      <td>60.69</td>\n",
       "      <td>22.35</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>8</td>\n",
       "      <td>B</td>\n",
       "      <td>557974</td>\n",
       "      <td>557303</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>24566</td>\n",
       "      <td>24451</td>\n",
       "      <td>2474.8</td>\n",
       "      <td>43.01</td>\n",
       "      <td>29.94</td>\n",
       "      <td>72.96</td>\n",
       "      <td>19.26</td>\n",
       "      <td>7.78</td>\n",
       "      <td>24435</td>\n",
       "      <td>29.83</td>\n",
       "      <td>63.91</td>\n",
       "      <td>6.26</td>\n",
       "      <td>29.77</td>\n",
       "      <td>60.20</td>\n",
       "      <td>10.02</td>\n",
       "      <td>18.18</td>\n",
       "      <td>74.83</td>\n",
       "      <td>6.98</td>\n",
       "      <td>28.87</td>\n",
       "      <td>64.16</td>\n",
       "      <td>6.97</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>00</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023</td>\n",
       "      <td>28</td>\n",
       "      <td>B</td>\n",
       "      <td>22322</td>\n",
       "      <td>22290</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3265</td>\n",
       "      <td>3165</td>\n",
       "      <td>2344.5</td>\n",
       "      <td>7.30</td>\n",
       "      <td>10.30</td>\n",
       "      <td>17.60</td>\n",
       "      <td>21.04</td>\n",
       "      <td>61.36</td>\n",
       "      <td>3165</td>\n",
       "      <td>5.81</td>\n",
       "      <td>48.34</td>\n",
       "      <td>45.85</td>\n",
       "      <td>4.84</td>\n",
       "      <td>37.61</td>\n",
       "      <td>57.55</td>\n",
       "      <td>4.96</td>\n",
       "      <td>65.91</td>\n",
       "      <td>29.13</td>\n",
       "      <td>6.32</td>\n",
       "      <td>51.53</td>\n",
       "      <td>42.15</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269489</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>128</td>\n",
       "      <td>B</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.76</td>\n",
       "      <td>88.24</td>\n",
       "      <td>17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.76</td>\n",
       "      <td>88.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17.65</td>\n",
       "      <td>82.35</td>\n",
       "      <td>0.00</td>\n",
       "      <td>23.53</td>\n",
       "      <td>76.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269494</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2586.9</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269495</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>2550.0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>20</td>\n",
       "      <td>15.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>45.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269496</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>20</td>\n",
       "      <td>15.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>45.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3269497</th>\n",
       "      <td>58</td>\n",
       "      <td>72769</td>\n",
       "      <td>0000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>144</td>\n",
       "      <td>B</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>20</td>\n",
       "      <td>45.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>25.00</td>\n",
       "      <td>65.00</td>\n",
       "      <td>10.00</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524552 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        county_code district_code school_code filler test_year  \\\n",
       "0                00         00000     0000000    NaN      2023   \n",
       "1                00         00000     0000000    NaN      2023   \n",
       "2                00         00000     0000000    NaN      2023   \n",
       "5                00         00000     0000000    NaN      2023   \n",
       "6                00         00000     0000000    NaN      2023   \n",
       "...             ...           ...         ...    ...       ...   \n",
       "3269489          58         72769     0000000    NaN      2018   \n",
       "3269494          58         72769     0000000    NaN      2018   \n",
       "3269495          58         72769     0000000    NaN      2018   \n",
       "3269496          58         72769     0000000    NaN      2018   \n",
       "3269497          58         72769     0000000    NaN      2018   \n",
       "\n",
       "        student_group_id test_type total_tested_reporting_lvl  \\\n",
       "0                      1         B                    2960990   \n",
       "1                      3         B                    1512977   \n",
       "2                      4         B                    1445927   \n",
       "5                      8         B                     557974   \n",
       "6                     28         B                      22322   \n",
       "...                  ...       ...                        ...   \n",
       "3269489              128         B                         17   \n",
       "3269494              144         B                         20   \n",
       "3269495              144         B                         20   \n",
       "3269496              144         B                         20   \n",
       "3269497              144         B                         20   \n",
       "\n",
       "        total_tested_with_scores grade test_id students_enrolled  \\\n",
       "0                        2957491     3       1            420238   \n",
       "1                        1511175     3       1            214436   \n",
       "2                        1444241     3       1            205715   \n",
       "5                         557303     3       1             24566   \n",
       "6                          22290     3       1              3265   \n",
       "...                          ...   ...     ...               ...   \n",
       "3269489                       17    13       2                18   \n",
       "3269494                       20    11       1                20   \n",
       "3269495                       20    11       2                20   \n",
       "3269496                       20    13       2                20   \n",
       "3269497                       20    13       1                20   \n",
       "\n",
       "        students_tested mean_scale_score pct_std_exceeded pct_std_met  \\\n",
       "0                410559           2409.8            23.40       19.56   \n",
       "1                209138           2402.4            21.27       18.77   \n",
       "2                201341           2417.5            25.60       20.39   \n",
       "5                 24451           2474.8            43.01       29.94   \n",
       "6                  3165           2344.5             7.30       10.30   \n",
       "...                 ...              ...              ...         ...   \n",
       "3269489              17              NaN             0.00        0.00   \n",
       "3269494              20           2586.9            30.00       20.00   \n",
       "3269495              20           2550.0            10.00       20.00   \n",
       "3269496              20              NaN            10.00       20.00   \n",
       "3269497              20              NaN            30.00       20.00   \n",
       "\n",
       "        pct_std_met_above pct_std_nearly_met pct_std_not_met  \\\n",
       "0                   42.96              22.40           34.64   \n",
       "1                   40.04              22.45           37.51   \n",
       "2                   45.99              22.34           31.67   \n",
       "5                   72.96              19.26            7.78   \n",
       "6                   17.60              21.04           61.36   \n",
       "...                   ...                ...             ...   \n",
       "3269489              0.00              11.76           88.24   \n",
       "3269494             50.00              25.00           25.00   \n",
       "3269495             30.00              35.00           35.00   \n",
       "3269496             30.00              35.00           35.00   \n",
       "3269497             50.00              25.00           25.00   \n",
       "\n",
       "        students_with_scores area_1_pct_above_std area_1_pct_near_std  \\\n",
       "0                     410290                17.47               57.69   \n",
       "1                     208998                15.88               57.23   \n",
       "2                     201212                19.12               58.17   \n",
       "5                      24435                29.83               63.91   \n",
       "6                       3165                 5.81               48.34   \n",
       "...                      ...                  ...                 ...   \n",
       "3269489                   17                 0.00               11.76   \n",
       "3269494                   20                45.00               30.00   \n",
       "3269495                   20                15.00               40.00   \n",
       "3269496                   20                15.00               40.00   \n",
       "3269497                   20                45.00               30.00   \n",
       "\n",
       "        area_1_pct_below_std area_2_pct_above_std area_2_pct_near_std  \\\n",
       "0                      24.84                16.18               50.24   \n",
       "1                      26.89                13.96               49.35   \n",
       "2                      22.71                18.48               51.16   \n",
       "5                       6.26                29.77               60.20   \n",
       "6                      45.85                 4.84               37.61   \n",
       "...                      ...                  ...                 ...   \n",
       "3269489                88.24                 0.00               17.65   \n",
       "3269494                25.00                35.00               30.00   \n",
       "3269495                45.00                25.00               35.00   \n",
       "3269496                45.00                25.00               35.00   \n",
       "3269497                25.00                35.00               30.00   \n",
       "\n",
       "        area_2_pct_below_std area_3_pct_above_std area_3_pct_near_std  \\\n",
       "0                      33.58                11.75               70.96   \n",
       "1                      36.69                11.47               70.55   \n",
       "2                      30.36                12.04               71.39   \n",
       "5                      10.02                18.18               74.83   \n",
       "6                      57.55                 4.96               65.91   \n",
       "...                      ...                  ...                 ...   \n",
       "3269489                82.35                 0.00               23.53   \n",
       "3269494                35.00                25.00               65.00   \n",
       "3269495                40.00                25.00               35.00   \n",
       "3269496                40.00                25.00               35.00   \n",
       "3269497                35.00                25.00               65.00   \n",
       "\n",
       "        area_3_pct_below_std area_4_pct_above_std area_4_pct_near_std  \\\n",
       "0                      17.29                16.07               60.00   \n",
       "1                      17.99                15.20               59.33   \n",
       "2                      16.57                16.96               60.69   \n",
       "5                       6.98                28.87               64.16   \n",
       "6                      29.13                 6.32               51.53   \n",
       "...                      ...                  ...                 ...   \n",
       "3269489                76.47                 0.00                0.00   \n",
       "3269494                10.00                30.00               35.00   \n",
       "3269495                40.00                 0.00                0.00   \n",
       "3269496                40.00                 0.00                0.00   \n",
       "3269497                10.00                30.00               35.00   \n",
       "\n",
       "        area_4_pct_below_std type_id  is_wcc_entity  \n",
       "0                      23.94       4          False  \n",
       "1                      25.47       4          False  \n",
       "2                      22.35       4          False  \n",
       "5                       6.97       4          False  \n",
       "6                      42.15       4          False  \n",
       "...                      ...     ...            ...  \n",
       "3269489                 0.00       6          False  \n",
       "3269494                35.00       6          False  \n",
       "3269495                 0.00       6          False  \n",
       "3269496                 0.00       6          False  \n",
       "3269497                35.00       6          False  \n",
       "\n",
       "[524552 rows x 34 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Existing conditions for \"Aggregate Level\" and \"is_wcc_entity\"\n",
    "condition1 = final_caaspp_df['is_wcc_entity']\n",
    "\n",
    "# State Condition\n",
    "condition2 = final_caaspp_df['county_code'] == '00'\n",
    "\n",
    "# District Condition\n",
    "conditionA = final_caaspp_df['district_code'] != '00000'\n",
    "conditionB = final_caaspp_df['school_code'] == '0000000'\n",
    "\n",
    "condition3 = conditionA & conditionB\n",
    "\n",
    "# Student with Scores >= 11\n",
    "\n",
    "lower_counts = {'0' ,'1', '2', '3', '4' ,'5', '6', '7', '8', '9', '10', '*'}\n",
    "condition4 = ~final_caaspp_df['students_with_scores'].isin(lower_counts)\n",
    "\n",
    "combined_condition = (condition1 |  condition2 | condition3) & condition4 \n",
    "\n",
    "# Filter dataframe1 using the combined condition\n",
    "final_filtered_caaspp_df = final_caaspp_df[combined_condition]\n",
    "\n",
    "final_filtered_caaspp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_filtered_caaspp_df.to_csv('final_long_datasets_domain/final_caaspp_all_grades_df.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}