docs/tutorials/other_utilities.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "05cffd55-d251-4dce-9b0d-f8e12c8f2a43",
   "metadata": {},
   "source": [
    "# Other Utilities"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c3c63b5-3302-422d-9d79-503fb8b630cc",
   "metadata": {},
   "source": [
    "## Generating Full Span of multiple time-series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8fec6271-83bb-42bd-8d6f-e62dd0b2640d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from orbit.utils.general import expand_grid, regenerate_base_df\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "549b7ed0-6bc4-404b-a4d5-ae5a499115ac",
   "metadata": {},
   "source": [
    "Define the series keys and datetime array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6b2eb113-8a81-4cec-9698-2079d9052dc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9']\n",
      "DatetimeIndex(['2020-01-31', '2020-02-29', '2020-03-31', '2020-04-30',\n",
      "               '2020-05-31', '2020-06-30', '2020-07-31', '2020-08-31',\n",
      "               '2020-09-30', '2020-10-31', '2020-11-30', '2020-12-31',\n",
      "               '2021-01-31', '2021-02-28', '2021-03-31', '2021-04-30',\n",
      "               '2021-05-31', '2021-06-30', '2021-07-31', '2021-08-31',\n",
      "               '2021-09-30', '2021-10-31', '2021-11-30', '2021-12-31',\n",
      "               '2022-01-31', '2022-02-28', '2022-03-31', '2022-04-30',\n",
      "               '2022-05-31', '2022-06-30', '2022-07-31', '2022-08-31',\n",
      "               '2022-09-30', '2022-10-31', '2022-11-30', '2022-12-31'],\n",
      "              dtype='datetime64[ns]', freq='ME')\n"
     ]
    }
   ],
   "source": [
    "dt = pd.date_range('2020-01-31', '2022-12-31', freq='M')\n",
    "keys = ['x' + str(x) for x in range(10)]\n",
    "print(keys)\n",
    "print(dt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "355af3a9-9291-4888-bd98-a7d0f093b100",
   "metadata": {},
   "source": [
    "Users can use `expand_grid` to generate dataframe with observations in `key` and `dt` levels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0b05ade8-6d99-4ccd-8375-b574711e1bf1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(360, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>dt</th>\n",
       "      <th>x</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>0.357236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-02-29</td>\n",
       "      <td>-1.172618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>-0.852877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>1.080290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>-0.641044</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  key         dt         x\n",
       "0  x0 2020-01-31  0.357236\n",
       "1  x0 2020-02-29 -1.172618\n",
       "2  x0 2020-03-31 -0.852877\n",
       "3  x0 2020-04-30  1.080290\n",
       "4  x0 2020-05-31 -0.641044"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_base = expand_grid({\n",
    "    'key': keys,\n",
    "    'dt': dt,\n",
    "})\n",
    "x = np.random.normal(0, 1, 10 * 36)\n",
    "df_base['x'] = x\n",
    "print(df_base.shape)\n",
    "df_base.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3417232-1734-4203-a373-2925f88f4f81",
   "metadata": {},
   "source": [
    "## Regenerate Multiple Timeseries with Missing rows"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0bb9525-c284-40aa-adcd-81a932db97de",
   "metadata": {},
   "source": [
    "Create missing rows."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "94da7385-6286-4f8a-ba48-603bdf6e4457",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(355, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>dt</th>\n",
       "      <th>x</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>0.357236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-02-29</td>\n",
       "      <td>-1.172618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>-0.852877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>1.080290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>x0</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>-0.641044</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  key         dt         x\n",
       "0  x0 2020-01-31  0.357236\n",
       "1  x0 2020-02-29 -1.172618\n",
       "2  x0 2020-03-31 -0.852877\n",
       "3  x0 2020-04-30  1.080290\n",
       "4  x0 2020-05-31 -0.641044"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(2022)\n",
    "drop_idx = np.random.choice(df_base.index, 5, replace=False)\n",
    "df_missing = df_base.drop(drop_idx).reset_index(drop=True)\n",
    "print(df_missing.shape)\n",
    "df_missing.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62222dd2-b150-442a-bd8d-79916ab6e87e",
   "metadata": {},
   "source": [
    "Use `regenerate_base_df` to regenerate the base dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d637961-5fda-435b-b9c8-d20512a67f16",
   "metadata": {},
   "outputs": [],
   "source": [
    "time_col = \"dt\"\n",
    "key_col = \"key\"\n",
    "new_df_base = regenerate_base_df(df_missing, time_col, key_col, val_cols=['x'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89faea4c-a974-4815-89aa-c55f3b1f78d8",
   "metadata": {},
   "source": [
    "By default, the missing entries regenerated come with a null value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cfc7788e-3412-4531-a77a-3b543643c02a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dt</th>\n",
       "      <th>key</th>\n",
       "      <th>x</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>2022-11-30</td>\n",
       "      <td>x7</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>2021-11-30</td>\n",
       "      <td>x7</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>x2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>2022-04-30</td>\n",
       "      <td>x3</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>2020-08-31</td>\n",
       "      <td>x1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            dt key   x\n",
       "286 2022-11-30  x7 NaN\n",
       "274 2021-11-30  x7 NaN\n",
       "75  2020-04-30  x2 NaN\n",
       "135 2022-04-30  x3 NaN\n",
       "43  2020-08-31  x1 NaN"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df_base.iloc[drop_idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daa9b7d1-4241-46c0-a49e-c8d4fa2a7938",
   "metadata": {},
   "source": [
    "Users can also use `fill_na` option to fill the missing values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1c26df77-010e-4d7b-9f23-b5a8f169b4c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df_base = regenerate_base_df(df_missing, time_col, key_col, val_cols=['x'], fill_na=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ff095409-61e6-4049-964f-509765fe2665",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dt</th>\n",
       "      <th>key</th>\n",
       "      <th>x</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>2022-11-30</td>\n",
       "      <td>x7</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>2021-11-30</td>\n",
       "      <td>x7</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>x2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>2022-04-30</td>\n",
       "      <td>x3</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>2020-08-31</td>\n",
       "      <td>x1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            dt key    x\n",
       "286 2022-11-30  x7  0.0\n",
       "274 2021-11-30  x7  0.0\n",
       "75  2020-04-30  x2  0.0\n",
       "135 2022-04-30  x3  0.0\n",
       "43  2020-08-31  x1  0.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df_base.iloc[drop_idx]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "orbit311",
   "language": "python",
   "name": "orbit311"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}