S1_BERT_plotting.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q7L5plh010Hs"
      },
      "source": [
        "# Question-Answering Machine Learning Model (DistilBERT)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nr72yJ0J10Hx"
      },
      "source": [
        "## Download SQuADv2.0 Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jLCQ689710Hx",
        "outputId": "4b1e0f7c-d821-4727-d65c-89bdcfd312f2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-06-23 00:47:03--  https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json\n",
            "Resolving rajpurkar.github.io (rajpurkar.github.io)... 185.199.108.153, 185.199.110.153, 185.199.111.153, ...\n",
            "Connecting to rajpurkar.github.io (rajpurkar.github.io)|185.199.108.153|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 42123633 (40M) [application/json]\n",
            "Saving to: ‘train-v2.0.json’\n",
            "\n",
            "train-v2.0.json     100%[===================>]  40.17M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2022-06-23 00:47:04 (291 MB/s) - ‘train-v2.0.json’ saved [42123633/42123633]\n",
            "\n",
            "--2022-06-23 00:47:04--  http://train-v2.0.json/\n",
            "Resolving train-v2.0.json (train-v2.0.json)... failed: Name or service not known.\n",
            "wget: unable to resolve host address ‘train-v2.0.json’\n",
            "FINISHED --2022-06-23 00:47:04--\n",
            "Total wall clock time: 1.3s\n",
            "Downloaded: 1 files, 40M in 0.1s (291 MB/s)\n",
            "--2022-06-23 00:47:04--  https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json\n",
            "Resolving rajpurkar.github.io (rajpurkar.github.io)... 185.199.108.153, 185.199.110.153, 185.199.111.153, ...\n",
            "Connecting to rajpurkar.github.io (rajpurkar.github.io)|185.199.108.153|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4854279 (4.6M) [application/json]\n",
            "Saving to: ‘dev-v1.1.json’\n",
            "\n",
            "dev-v1.1.json       100%[===================>]   4.63M  --.-KB/s    in 0.05s   \n",
            "\n",
            "2022-06-23 00:47:04 (85.5 MB/s) - ‘dev-v1.1.json’ saved [4854279/4854279]\n",
            "\n",
            "--2022-06-23 00:47:04--  http://dev-v2.0.json/\n",
            "Resolving dev-v2.0.json (dev-v2.0.json)... failed: Name or service not known.\n",
            "wget: unable to resolve host address ‘dev-v2.0.json’\n",
            "FINISHED --2022-06-23 00:47:04--\n",
            "Total wall clock time: 0.1s\n",
            "Downloaded: 1 files, 4.6M in 0.05s (85.5 MB/s)\n"
          ]
        }
      ],
      "source": [
        "!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json train-v2.0.json\n",
        "!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json dev-v2.0.json"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fipQiZ5YNX0d",
        "outputId": "875624ac-d26a-48bd-de7c-d0993fe3b722"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aSttiw1v10Hz",
        "outputId": "5289eabd-431c-4a53-8b3e-159867fbabe5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "wget is already the newest version (1.19.4-1ubuntu2.2).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 49 not upgraded.\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers==3.5 in /usr/local/lib/python3.7/dist-packages (3.5.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (2022.6.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (1.18.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (21.3)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (3.17.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (2.23.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (0.0.53)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (3.7.1)\n",
            "Requirement already satisfied: tokenizers==0.9.3 in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (0.9.3)\n",
            "Requirement already satisfied: sentencepiece==0.1.91 in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (0.1.91)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==3.5) (4.64.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==3.5) (3.0.9)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.7/dist-packages (from protobuf->transformers==3.5) (1.15.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.5) (2022.6.15)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.5) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.5) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.5) (1.24.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.5) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.5) (1.1.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: tensorflow==2.3.1 in /usr/local/lib/python3.7/dist-packages (2.3.1)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (2.10.0)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (0.3.3)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (3.3.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (0.2.0)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (1.6.3)\n",
            "Requirement already satisfied: tensorboard<3,>=2.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (2.8.0)\n",
            "Requirement already satisfied: tensorflow-estimator<2.4.0,>=2.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (2.3.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (3.17.3)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (1.15.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (1.14.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (0.37.1)\n",
            "Requirement already satisfied: numpy<1.19.0,>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (1.18.5)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (1.46.3)\n",
            "Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (1.1.2)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.1) (1.1.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.1) (3.3.7)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.1) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.1) (1.8.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.1) (1.0.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.1) (1.35.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.1) (57.4.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.1) (0.6.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.1) (0.4.6)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (4.2.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (4.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (4.11.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (3.8.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (4.1.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (2022.6.15)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (1.24.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow==2.3.1) (3.2.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torch==1.6 in /usr/local/lib/python3.7/dist-packages (1.6.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch==1.6) (1.18.5)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from torch==1.6) (0.16.0)\n"
          ]
        }
      ],
      "source": [
        "!apt-get install wget\n",
        "!pip install transformers==3.5\n",
        "!pip install tensorflow==2.3.1\n",
        "!pip install torch==1.6\n",
        "!pip install -Uqq ipdb\n",
        "import ipdb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F7kWzbVC10Hz"
      },
      "outputs": [],
      "source": [
        "import argparse\n",
        "import glob\n",
        "import logging\n",
        "import os\n",
        "import random\n",
        "import timeit\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch\n",
        "import tensorflow as tf\n",
        "\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from torch.utils.data.distributed import DistributedSampler\n",
        "from tqdm import tqdm, trange\n",
        "\n",
        "from transformers import (\n",
        "    MODEL_FOR_QUESTION_ANSWERING_MAPPING,\n",
        "    WEIGHTS_NAME,\n",
        "    AdamW,\n",
        "    AutoConfig,\n",
        "    AutoModelForQuestionAnswering,\n",
        "    AutoTokenizer,\n",
        "    get_linear_schedule_with_warmup,\n",
        "    squad_convert_examples_to_features,\n",
        ")\n",
        "from transformers.data.metrics.squad_metrics import (\n",
        "    compute_predictions_log_probs,\n",
        "    compute_predictions_logits,\n",
        "    squad_evaluate,\n",
        ")\n",
        "from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor\n",
        "from torch.utils.data import TensorDataset\n",
        "\n",
        "try:\n",
        "    from torch.utils.tensorboard import SummaryWriter\n",
        "except ImportError:\n",
        "    from tensorboardX import SummaryWriter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fj-8BfoZ10H0"
      },
      "outputs": [],
      "source": [
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "MODEL_CONFIG_CLASSES = list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())\n",
        "MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HCbw_N7010H0"
      },
      "source": [
        "## DATASET FRACTION"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_nyPPfZe10H1"
      },
      "outputs": [],
      "source": [
        "coef = 0.2\n",
        "\n",
        "import json\n",
        "\n",
        "# Opening JSON file\n",
        "f = open('./dev-v1.1.json')\n",
        "  \n",
        "# returns JSON object as \n",
        "# a dictionary\n",
        "data = json.load(f)\n",
        "\n",
        "length = len(data['data'])\n",
        "\n",
        "newlen = int(length * coef)\n",
        "\n",
        "data_frag = random.sample(data['data'], newlen)\n",
        "\n",
        "newdata = {\"data\": data_frag}\n",
        "\n",
        "with open('./dev-v2.0.json', 'w', encoding='utf-8') as f1:\n",
        "    json.dump(newdata, f1, ensure_ascii=False, indent=4)\n",
        "\n",
        "f.close()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Helper functions"
      ],
      "metadata": {
        "id": "8DR5UTUbThHB"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DhHZ7lo43r4x"
      },
      "source": [
        "## MODEL_FOR_QUESTION_ANSWERING_MAPPING"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4mOT_UQg10H2"
      },
      "outputs": [],
      "source": [
        "def set_seed(args):\n",
        "    random.seed(args.seed)\n",
        "    np.random.seed(args.seed)\n",
        "    torch.manual_seed(args.seed)\n",
        "    if args.n_gpu > 0:\n",
        "        torch.cuda.manual_seed_all(args.seed)\n",
        "\n",
        "\n",
        "def to_list(tensor):\n",
        "    return tensor.detach().cpu().tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "quAlxafy10H2"
      },
      "outputs": [],
      "source": [
        "def train(args, train_dataset, model, tokenizer):\n",
        "    \"\"\" Train the model \"\"\"\n",
        "    if args.local_rank in [-1, 0]:\n",
        "        tb_writer = SummaryWriter()\n",
        "\n",
        "    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)\n",
        "    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)\n",
        "    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)\n",
        "\n",
        "    if args.max_steps > 0:\n",
        "        t_total = args.max_steps\n",
        "        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1\n",
        "    else:\n",
        "        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs\n",
        "\n",
        "    # Prepare optimizer and schedule (linear warmup and decay)\n",
        "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
        "    optimizer_grouped_parameters = [\n",
        "        {\n",
        "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
        "            \"weight_decay\": args.weight_decay,\n",
        "        },\n",
        "        {\"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], \"weight_decay\": 0.0},\n",
        "    ]\n",
        "    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)\n",
        "    scheduler = get_linear_schedule_with_warmup(\n",
        "        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total\n",
        "    )\n",
        "\n",
        "    if args.fp16:\n",
        "        try:\n",
        "            from apex import amp\n",
        "        except ImportError:\n",
        "            raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to use fp16 training.\")\n",
        "\n",
        "        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)\n",
        "\n",
        "    # multi-gpu training (should be after apex fp16 initialization)\n",
        "    if args.n_gpu > 1:\n",
        "        model = torch.nn.DataParallel(model)\n",
        "\n",
        "    # Distributed training (should be after apex fp16 initialization)\n",
        "    if args.local_rank != -1:\n",
        "        model = torch.nn.parallel.DistributedDataParallel(\n",
        "            model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True\n",
        "        )\n",
        "\n",
        "    # Train!\n",
        "    logger.info(\"***** Running training *****\")\n",
        "    logger.info(\"  Num examples = %d\", len(train_dataset))\n",
        "    logger.info(\"  Num Epochs = %d\", args.num_train_epochs)\n",
        "    logger.info(\"  Instantaneous batch size per GPU = %d\", args.per_gpu_train_batch_size)\n",
        "    logger.info(\n",
        "        \"  Total train batch size (w. parallel, distributed & accumulation) = %d\",\n",
        "        args.train_batch_size\n",
        "        * args.gradient_accumulation_steps\n",
        "        * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),\n",
        "    )\n",
        "    logger.info(\"  Gradient Accumulation steps = %d\", args.gradient_accumulation_steps)\n",
        "    logger.info(\"  Total optimization steps = %d\", t_total)\n",
        "\n",
        "    global_step = 1\n",
        "    epochs_trained = 0\n",
        "    steps_trained_in_current_epoch = 0\n",
        "    # Check if continuing training from a checkpoint\n",
        "    if os.path.exists(args.model_name_or_path):\n",
        "        try:\n",
        "            # set global_step to gobal_step of last saved checkpoint from model path\n",
        "            checkpoint_suffix = args.model_name_or_path.split(\"-\")[-1].split(\"/\")[0]\n",
        "            global_step = int(checkpoint_suffix)\n",
        "            epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps)\n",
        "            steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps)\n",
        "\n",
        "            logger.info(\"  Continuing training from checkpoint, will skip to saved global_step\")\n",
        "            logger.info(\"  Continuing training from epoch %d\", epochs_trained)\n",
        "            logger.info(\"  Continuing training from global step %d\", global_step)\n",
        "            logger.info(\"  Will skip the first %d steps in the first epoch\", steps_trained_in_current_epoch)\n",
        "        except ValueError:\n",
        "            logger.info(\"  Starting fine-tuning.\")\n",
        "\n",
        "    tr_loss, logging_loss = 0.0, 0.0\n",
        "    model.zero_grad()\n",
        "    train_iterator = trange(\n",
        "        epochs_trained, int(args.num_train_epochs), desc=\"Epoch\", disable=args.local_rank not in [-1, 0]\n",
        "    )\n",
        "    # Added here for reproductibility\n",
        "    set_seed(args)\n",
        "\n",
        "    for _ in train_iterator:\n",
        "        epoch_iterator = tqdm(train_dataloader, desc=\"Iteration\", disable=args.local_rank not in [-1, 0])\n",
        "        for step, batch in enumerate(epoch_iterator):\n",
        "\n",
        "            # Skip past any already trained steps if resuming training\n",
        "            if steps_trained_in_current_epoch > 0:\n",
        "                steps_trained_in_current_epoch -= 1\n",
        "                continue\n",
        "\n",
        "            model.train()\n",
        "            batch = tuple(t.to(args.device) for t in batch)\n",
        "\n",
        "            inputs = {\n",
        "                \"input_ids\": batch[0],\n",
        "                \"attention_mask\": batch[1],\n",
        "                \"token_type_ids\": batch[2],\n",
        "                \"start_positions\": batch[3],\n",
        "                \"end_positions\": batch[4],\n",
        "            }\n",
        "\n",
        "            if args.model_type in [\"xlm\", \"roberta\", \"distilbert\", \"camembert\", \"bart\"]:\n",
        "                del inputs[\"token_type_ids\"]\n",
        "\n",
        "\n",
        "            outputs = model(**inputs)\n",
        "            # model outputs are always tuple in transformers (see doc)\n",
        "            loss = outputs[0]\n",
        "\n",
        "            if args.n_gpu > 1:\n",
        "                loss = loss.mean()  # mean() to average on multi-gpu parallel (not distributed) training\n",
        "            if args.gradient_accumulation_steps > 1:\n",
        "                loss = loss / args.gradient_accumulation_steps\n",
        "\n",
        "            tr_loss += loss.item()\n",
        "\n",
        "            if (step + 1) % args.gradient_accumulation_steps == 0:\n",
        "\n",
        "                optimizer.step()\n",
        "                scheduler.step()  # Update learning rate schedule\n",
        "                model.zero_grad()\n",
        "                global_step += 1\n",
        "\n",
        "                # Log metrics\n",
        "                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:\n",
        "                    # Only evaluate when single GPU otherwise metrics may not average well\n",
        "                    if args.local_rank == -1 and args.evaluate_during_training:\n",
        "                        results = evaluate(args, model, tokenizer)\n",
        "                        for key, value in results.items():\n",
        "                            tb_writer.add_scalar(\"eval_{}\".format(key), value, global_step)\n",
        "                    tb_writer.add_scalar(\"lr\", scheduler.get_lr()[0], global_step)\n",
        "                    tb_writer.add_scalar(\"loss\", (tr_loss - logging_loss) / args.logging_steps, global_step)\n",
        "                    logging_loss = tr_loss\n",
        "\n",
        "            if args.max_steps > 0 and global_step > args.max_steps:\n",
        "                epoch_iterator.close()\n",
        "                break\n",
        "        if args.max_steps > 0 and global_step > args.max_steps:\n",
        "            train_iterator.close()\n",
        "            break\n",
        "\n",
        "    if args.local_rank in [-1, 0]:\n",
        "        tb_writer.close()\n",
        "\n",
        "    return global_step, tr_loss / global_step\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## eval"
      ],
      "metadata": {
        "id": "7LzU3mFTc61u"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lB3XMjEj10H3"
      },
      "outputs": [],
      "source": [
        "def evaluate(args, model, tokenizer, train_dataset):\n",
        "    dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True)\n",
        "\n",
        "    if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:\n",
        "        os.makedirs(args.output_dir)\n",
        "\n",
        "    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)\n",
        "\n",
        "    # Note that DistributedSampler samples randomly\n",
        "    eval_sampler = SequentialSampler(dataset)\n",
        "    eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)\n",
        "\n",
        "    # multi-gpu evaluate\n",
        "    if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):\n",
        "        model = torch.nn.DataParallel(model)\n",
        "\n",
        "    # Eval!\n",
        "    logger.info(\"***** Running evaluation {} *****\".format(prefix))\n",
        "    logger.info(\"  Num examples = %d\", len(dataset))\n",
        "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
        "\n",
        "    all_results = []\n",
        "    start_time = timeit.default_timer()\n",
        "\n",
        "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
        "        model.eval()\n",
        "        batch = tuple(t.to(args.device) for t in batch)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            inputs = {\n",
        "                \"input_ids\": batch[0],\n",
        "                \"attention_mask\": batch[1],\n",
        "                \"token_type_ids\": batch[2],\n",
        "            }\n",
        "\n",
        "            if args.model_type in [\"xlm\", \"roberta\", \"distilbert\", \"camembert\", \"bart\"]:\n",
        "                del inputs[\"token_type_ids\"]\n",
        "\n",
        "            feature_indices = batch[3]\n",
        "\n",
        "            # XLNet and XLM use more arguments for their predictions\n",
        "            if args.model_type in [\"xlnet\", \"xlm\"]:\n",
        "                inputs.update({\"cls_index\": batch[4], \"p_mask\": batch[5]})\n",
        "                # for lang_id-sensitive xlm models\n",
        "                if hasattr(model, \"config\") and hasattr(model.config, \"lang2id\"):\n",
        "                    inputs.update(\n",
        "                        {\"langs\": (torch.ones(batch[0].shape, dtype=torch.int64) * args.lang_id).to(args.device)}\n",
        "                    )\n",
        "                    \n",
        "            outputs = model(**inputs)\n",
        "\n",
        "        for i, feature_index in enumerate(feature_indices):\n",
        "            eval_feature = features[feature_index.item()]\n",
        "            unique_id = int(eval_feature.unique_id)\n",
        "\n",
        "            output = [to_list(output[i]) for output in outputs]\n",
        "\n",
        "            # Some models (XLNet, XLM) use 5 arguments for their predictions, while the other \"simpler\"\n",
        "            # models only use two.\n",
        "            if len(output) >= 5:\n",
        "                start_logits = output[0]\n",
        "                start_top_index = output[1]\n",
        "                end_logits = output[2]\n",
        "                end_top_index = output[3]\n",
        "                cls_logits = output[4]\n",
        "\n",
        "                result = SquadResult(\n",
        "                    unique_id,\n",
        "                    start_logits,\n",
        "                    end_logits,\n",
        "                    start_top_index=start_top_index,\n",
        "                    end_top_index=end_top_index,\n",
        "                    cls_logits=cls_logits,\n",
        "                )\n",
        "\n",
        "            else:\n",
        "                start_logits, end_logits = output\n",
        "                result = SquadResult(unique_id, start_logits, end_logits)\n",
        "\n",
        "            all_results.append(result)\n",
        "\n",
        "    evalTime = timeit.default_timer() - start_time\n",
        "    logger.info(\"  Evaluation done in total %f secs (%f sec per example)\", evalTime, evalTime / len(dataset))\n",
        "\n",
        "    # Compute predictions\n",
        "    output_prediction_file = os.path.join(args.output_dir, \"predictions_{}.json\".format(prefix))\n",
        "    output_nbest_file = os.path.join(args.output_dir, \"nbest_predictions_{}.json\".format(prefix))\n",
        "\n",
        "    if args.version_2_with_negative:\n",
        "        output_null_log_odds_file = os.path.join(args.output_dir, \"null_odds_{}.json\".format(prefix))\n",
        "    else:\n",
        "        output_null_log_odds_file = None\n",
        "\n",
        "    # XLNet and XLM use a more complex post-processing procedure\n",
        "    if args.model_type in [\"xlnet\", \"xlm\"]:\n",
        "        start_n_top = model.config.start_n_top if hasattr(model, \"config\") else model.module.config.start_n_top\n",
        "        end_n_top = model.config.end_n_top if hasattr(model, \"config\") else model.module.config.end_n_top\n",
        "\n",
        "        predictions = compute_predictions_log_probs(\n",
        "            examples,\n",
        "            features,\n",
        "            all_results,\n",
        "            args.n_best_size,\n",
        "            args.max_answer_length,\n",
        "            output_prediction_file,\n",
        "            output_nbest_file,\n",
        "            output_null_log_odds_file,\n",
        "            start_n_top,\n",
        "            end_n_top,\n",
        "            args.version_2_with_negative,\n",
        "            tokenizer,\n",
        "            args.verbose_logging,\n",
        "        )\n",
        "    else:\n",
        "        predictions = compute_predictions_logits(\n",
        "            examples,\n",
        "            features,\n",
        "            all_results,\n",
        "            args.n_best_size,\n",
        "            args.max_answer_length,\n",
        "            args.do_lower_case,\n",
        "            output_prediction_file,\n",
        "            output_nbest_file,\n",
        "            output_null_log_odds_file,\n",
        "            args.verbose_logging,\n",
        "            args.version_2_with_negative,\n",
        "            args.null_score_diff_threshold,\n",
        "            tokenizer,\n",
        "        )\n",
        "\n",
        "    # Compute the F1 and exact scores.\n",
        "    results = squad_evaluate(examples, predictions)\n",
        "    return results\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Dataload"
      ],
      "metadata": {
        "id": "91EAevf1WigE"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rWKhE5Vv10H4"
      },
      "outputs": [],
      "source": [
        "def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False):\n",
        "    if args.local_rank not in [-1, 0] and not evaluate:\n",
        "        # Make sure only the first process in distributed training process the dataset, and the others will use the cache\n",
        "        torch.distributed.barrier()\n",
        "\n",
        "    # Load data features from cache or dataset file\n",
        "    input_dir = args.data_dir if args.data_dir else \".\"\n",
        "    cached_features_file = os.path.join(\n",
        "        input_dir,\n",
        "        \"cached_{}_{}_{}\".format(\n",
        "            \"dev\" if evaluate else \"train\",\n",
        "            list(filter(None, args.model_name_or_path.split(\"/\"))).pop(),\n",
        "            str(args.max_seq_length),\n",
        "        ),\n",
        "    )\n",
        "    root_dir = os.path.join(cached_features_file+\"_dir\")\n",
        "    features_file = os.path.join(root_dir,'features')\n",
        "    datasets_file = os.path.join(root_dir,'datasets')\n",
        "    examples_file = os.path.join(root_dir,'examples')\n",
        "\n",
        "    # Init features and dataset from cache if it exists\n",
        "    if os.path.exists(cached_features_file) and not args.overwrite_cache:\n",
        "        logger.info(\"Loading features from cached file %s\", cached_features_file)\n",
        "        features_and_dataset = torch.load(cached_features_file)\n",
        "        features, dataset, examples = (\n",
        "            features_and_dataset[\"features\"],\n",
        "            features_and_dataset[\"dataset\"],\n",
        "            features_and_dataset[\"examples\"],\n",
        "        )\n",
        "        if output_examples:\n",
        "            return features, dataset, examples\n",
        "        else:\n",
        "            return dataset\n",
        "    elif os.path.exists(datasets_file) and not output_examples and not args.overwrite_cache:\n",
        "        pass\n",
        "    elif os.path.exists(datasets_file) and os.path.exists(features_file) and os.path.exists(examples_file) and output_examples and not args.overwrite_cache:\n",
        "        pass\n",
        "    else:\n",
        "        logger.info(\"Creating features from dataset file at %s\", input_dir)\n",
        "\n",
        "        if not args.data_dir and ((evaluate and not args.predict_file) or (not evaluate and not args.train_file)):\n",
        "            try:\n",
        "                import tensorflow_datasets as tfds\n",
        "            except ImportError:\n",
        "                raise ImportError(\"If not data_dir is specified, tensorflow_datasets needs to be installed.\")\n",
        "\n",
        "            if args.version_2_with_negative:\n",
        "                logger.warn(\"tensorflow_datasets does not handle version 2 of SQuAD.\")\n",
        "\n",
        "            tfds_examples = tfds.load(\"squad\")\n",
        "            examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate)\n",
        "        else:\n",
        "            processor = SquadV2Processor() if args.version_2_with_negative else SquadV1Processor()\n",
        "            if evaluate:\n",
        "                examples = processor.get_dev_examples(args.data_dir, filename=args.predict_file)\n",
        "            else:\n",
        "                examples = processor.get_train_examples(args.data_dir, filename=args.train_file)\n",
        "        \n",
        "        args.data_process_batch = args.data_process_batch if args.data_process_batch>0 else len(examples)\n",
        "        for i,j in enumerate(range(0,len(examples),args.data_process_batch)):\n",
        "            sub_examples = examples[j:j+args.data_process_batch]\n",
        "            features, dataset = squad_convert_examples_to_features(\n",
        "                examples=sub_examples,\n",
        "                tokenizer=tokenizer,\n",
        "                max_seq_length=args.max_seq_length,\n",
        "                doc_stride=args.doc_stride,\n",
        "                max_query_length=args.max_query_length,\n",
        "                is_training=not evaluate,\n",
        "                return_dataset=\"pt\",\n",
        "                threads=args.threads,\n",
        "            )\n",
        "\n",
        "            if args.local_rank in [-1, 0]:\n",
        "                if not os.path.exists(os.path.join(features_file)):\n",
        "                    os.makedirs(os.path.join(features_file))\n",
        "                if not os.path.exists(os.path.join(datasets_file)):\n",
        "                    os.makedirs(os.path.join(datasets_file))\n",
        "                if not os.path.exists(os.path.join(examples_file)):\n",
        "                    os.makedirs(os.path.join(examples_file))\n",
        "\n",
        "                logger.info(\"Saving features into cached files %s, %s, %s\", os.path.join(features_file,'features_'+str(i)),os.path.join(datasets_file,'datasets_'+str(i)),os.path.join(examples_file,'examples_'+str(i)))\n",
        "                torch.save({\"features\": features}, os.path.join(features_file,'features_'+str(i)))\n",
        "                torch.save({\"datasets\": dataset}, os.path.join(datasets_file,'datasets_'+str(i)))\n",
        "                torch.save({\"examples\": sub_examples}, os.path.join(examples_file,'examples_'+str(i)))\n",
        "\n",
        "    if args.local_rank == 0 and not evaluate:\n",
        "        # Make sure only the first process in distributed training process the dataset, and the others will use the cache\n",
        "        torch.distributed.barrier()\n",
        "    return read_saved_data(root_dir,evaluate=evaluate,output_examples=output_examples)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3Udb8SF410H5"
      },
      "outputs": [],
      "source": [
        "def read_saved_data(input_dir,evaluate=False,output_examples=False):\n",
        "    from torch.utils.data import TensorDataset\n",
        "\n",
        "    if output_examples:\n",
        "        feat=\"features;datasets;examples\"\n",
        "    else:\n",
        "        feat=\"datasets\"\n",
        "\n",
        "    all_features = {\"features\":[],\"examples\":[],\"datasets\":[]}\n",
        "    all_input_ids = torch.tensor([], dtype=torch.long)\n",
        "    all_attention_masks = torch.tensor([], dtype=torch.long)\n",
        "    all_token_type_ids = torch.tensor([], dtype=torch.long)\n",
        "    all_cls_index = torch.tensor([], dtype=torch.long)\n",
        "    all_p_mask = torch.tensor([], dtype=torch.float)\n",
        "    all_is_impossible = torch.tensor([], dtype=torch.float)\n",
        "    all_start_positions = torch.tensor([], dtype=torch.long)\n",
        "    all_end_positions = torch.tensor([], dtype=torch.long)\n",
        "\n",
        "    for i in feat.split(\";\"):\n",
        "        for file_name in os.listdir(os.path.join(input_dir,i)):\n",
        "            data = torch.load(os.path.join(input_dir,i,file_name))[i]\n",
        "            if isinstance(data,TensorDataset):\n",
        "                if evaluate:\n",
        "                    all_input_ids = torch.cat([all_input_ids,data.tensors[0]],dim=0)\n",
        "                    all_attention_masks = torch.cat([all_attention_masks,data.tensors[1]],dim=0)\n",
        "                    all_token_type_ids = torch.cat([all_token_type_ids,data.tensors[2]],dim=0)\n",
        "                    all_cls_index = torch.cat([all_cls_index,data.tensors[4]],dim=0)\n",
        "                    all_p_mask = torch.cat([all_p_mask,data.tensors[5]],dim=0)\n",
        "\n",
        "                    #all_start_positions = torch.cat([all_start_positions,data.tensors[6]],dim=0)\n",
        "                    #all_end_positions = torch.cat([all_end_positions,data.tensors[7]],dim=0)\n",
        "                else:\n",
        "                    all_input_ids = torch.cat([all_input_ids,data.tensors[0]],dim=0)\n",
        "                    all_attention_masks = torch.cat([all_attention_masks,data.tensors[1]],dim=0)\n",
        "                    all_token_type_ids = torch.cat([all_token_type_ids,data.tensors[2]],dim=0)\n",
        "                    all_start_positions = torch.cat([all_start_positions,data.tensors[3]],dim=0)\n",
        "                    all_end_positions = torch.cat([all_end_positions,data.tensors[4]],dim=0)\n",
        "                    all_cls_index = torch.cat([all_cls_index,data.tensors[5]],dim=0)\n",
        "                    all_p_mask = torch.cat([all_p_mask,data.tensors[6]],dim=0)\n",
        "                    all_is_impossible = torch.cat([all_is_impossible,data.tensors[7]],dim=0)\n",
        "            elif isinstance(data,list):\n",
        "                all_features[i] += data\n",
        "    \n",
        "    if evaluate and \"datasets\" in feat.split(\";\"):\n",
        "        all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)\n",
        "        all_features[\"datasets\"] = TensorDataset(all_input_ids, all_attention_masks, all_token_type_ids, all_example_index, all_cls_index, all_p_mask)\n",
        "    elif not evaluate and \"datasets\" in feat.split(\";\"):\n",
        "        all_features[\"datasets\"] = TensorDataset(all_input_ids,all_attention_masks,all_token_type_ids,all_start_positions,all_end_positions,all_cls_index,all_p_mask,all_is_impossible,)\n",
        "\n",
        "\n",
        "    if output_examples:\n",
        "        return all_features['datasets'], all_features['examples'], all_features['features']\n",
        "    else:\n",
        "        return all_features['datasets']\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VcVQTBCX2Rm9"
      },
      "source": [
        "## Arguments setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HNFxadjH10H5"
      },
      "outputs": [],
      "source": [
        "parser = argparse.ArgumentParser()\n",
        "\n",
        "parser.add_argument(\n",
        "    \"--model_type\",\n",
        "    type=str,\n",
        "    default=\"distilbert\",\n",
        "    help=\"Model type selected in the list: \" + \", \".join(MODEL_TYPES),\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--model_name_or_path\",\n",
        "    type=str,\n",
        "    default=\"distilbert-base-uncased\",\n",
        "    help=\"Path to pretrained model or model identifier from huggingface.co/models\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--output_dir\",\n",
        "    type=str,\n",
        "    default=\"./distilbert-squad-100\",\n",
        "    help=\"The output directory where the model checkpoints and predictions will be written.\",\n",
        ")\n",
        "\n",
        "# Other parameters\n",
        "parser.add_argument(\n",
        "    \"--data_dir\",\n",
        "    default=None,\n",
        "    type=str,\n",
        "    help=\"The input data dir. Should contain the .json files for the task.\"\n",
        "    + \"If no data dir or train/predict files are specified, will run with tensorflow_datasets.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--train_file\",\n",
        "    type=str,\n",
        "    default=\"train-v2.0.json\",\n",
        "    help=\"The input training file. If a data dir is specified, will look for the file there\"\n",
        "    + \"If no data dir or train/predict files are specified, will run with tensorflow_datasets.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--predict_file\",\n",
        "    type=str,\n",
        "    default='dev-v2.0.json',\n",
        "    help=\"The input evaluation file. If a data dir is specified, will look for the file there\"\n",
        "    + \"If no data dir or train/predict files are specified, will run with tensorflow_datasets.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--config_name\", default=\"\", type=str, help=\"Pretrained config name or path if not the same as model_name\"\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--tokenizer_name\",\n",
        "    default=\"\",\n",
        "    type=str,\n",
        "    help=\"Pretrained tokenizer name or path if not the same as model_name\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--cache_dir\",\n",
        "    default=\"\",\n",
        "    type=str,\n",
        "    help=\"Where do you want to store the pre-trained models downloaded from s3\",\n",
        ")\n",
        "\n",
        "parser.add_argument(\n",
        "    \"--version_2_with_negative\",\n",
        "    action=\"store_true\",\n",
        "    help=\"If true, the SQuAD examples contain some that do not have an answer.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--null_score_diff_threshold\",\n",
        "    type=float,\n",
        "    default=0.0,\n",
        "    help=\"If null_score - best_non_null is greater than the threshold predict null.\",\n",
        ")\n",
        "\n",
        "parser.add_argument(\n",
        "    \"--max_seq_length\",\n",
        "    default=384,\n",
        "    type=int,\n",
        "    help=\"The maximum total input sequence length after WordPiece tokenization. Sequences \"\n",
        "    \"longer than this will be truncated, and sequences shorter than this will be padded.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--doc_stride\",\n",
        "    default=128,\n",
        "    type=int,\n",
        "    help=\"When splitting up a long document into chunks, how much stride to take between chunks.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--max_query_length\",\n",
        "    default=64,\n",
        "    type=int,\n",
        "    help=\"The maximum number of tokens for the question. Questions longer than this will \"\n",
        "    \"be truncated to this length.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--data_process_batch\",\n",
        "    default=67000,\n",
        "    type=int,\n",
        "    help=\"Number of batches in which SQuAD data will be processed. If -1 provided, complete data will be processed at once.\",\n",
        ")\n",
        "parser.add_argument(\"--do_train\", action=\"store_true\", help=\"Whether to run training.\")\n",
        "parser.add_argument(\"--do_eval\", action=\"store_true\", help=\"Whether to run eval on the dev set.\")\n",
        "parser.add_argument(\n",
        "    \"--evaluate_during_training\", action=\"store_true\", help=\"Run evaluation during training at each logging step.\"\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--do_lower_case\", action=\"store_true\", help=\"Set this flag if you are using an uncased model.\"\n",
        ")\n",
        "\n",
        "parser.add_argument(\"--per_gpu_train_batch_size\", default=8, type=int, help=\"Batch size per GPU/CPU for training.\")\n",
        "parser.add_argument(\n",
        "    \"--per_gpu_eval_batch_size\", default=8, type=int, help=\"Batch size per GPU/CPU for evaluation.\"\n",
        ")\n",
        "parser.add_argument(\"--learning_rate\", default=5e-5, type=float, help=\"The initial learning rate for Adam.\")\n",
        "parser.add_argument(\n",
        "    \"--gradient_accumulation_steps\",\n",
        "    type=int,\n",
        "    default=1,\n",
        "    help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n",
        ")\n",
        "parser.add_argument(\"--weight_decay\", default=0.0, type=float, help=\"Weight decay if we apply some.\")\n",
        "parser.add_argument(\"--adam_epsilon\", default=1e-8, type=float, help=\"Epsilon for Adam optimizer.\")\n",
        "parser.add_argument(\"--max_grad_norm\", default=1.0, type=float, help=\"Max gradient norm.\")\n",
        "parser.add_argument(\n",
        "    \"--num_train_epochs\", default=3.0, type=float, help=\"Total number of training epochs to perform.\"\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--max_steps\",\n",
        "    default=-1,\n",
        "    type=int,\n",
        "    help=\"If > 0: set total number of training steps to perform. Override num_train_epochs.\",\n",
        ")\n",
        "parser.add_argument(\"--warmup_steps\", default=0, type=int, help=\"Linear warmup over warmup_steps.\")\n",
        "parser.add_argument(\n",
        "    \"--n_best_size\",\n",
        "    default=20,\n",
        "    type=int,\n",
        "    help=\"The total number of n-best predictions to generate in the nbest_predictions.json output file.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--max_answer_length\",\n",
        "    default=30,\n",
        "    type=int,\n",
        "    help=\"The maximum length of an answer that can be generated. This is needed because the start \"\n",
        "    \"and end predictions are not conditioned on one another.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--verbose_logging\",\n",
        "    action=\"store_true\",\n",
        "    help=\"If true, all of the warnings related to data processing will be printed. \"\n",
        "    \"A number of warnings are expected for a normal SQuAD evaluation.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--lang_id\",\n",
        "    default=0,\n",
        "    type=int,\n",
        "    help=\"language id of input for language-specific xlm models (see tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)\",\n",
        ")\n",
        "\n",
        "parser.add_argument(\"--logging_steps\", type=int, default=500, help=\"Log every X updates steps.\")\n",
        "parser.add_argument(\"--save_steps\", type=int, default=500, help=\"Save checkpoint every X updates steps.\")\n",
        "parser.add_argument(\n",
        "    \"--eval_all_checkpoints\",\n",
        "    action=\"store_true\",\n",
        "    help=\"Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number\",\n",
        ")\n",
        "parser.add_argument(\"--no_cuda\", action=\"store_true\", help=\"Whether not to use CUDA when available\")\n",
        "parser.add_argument(\n",
        "    \"--overwrite_output_dir\", default=True, action=\"store_true\", help=\"Overwrite the content of the output directory\"\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n",
        ")\n",
        "parser.add_argument(\"--seed\", type=int, default=42, help=\"random seed for initialization\")\n",
        "\n",
        "parser.add_argument(\"--local_rank\", type=int, default=-1, help=\"local_rank for distributed training on gpus\")\n",
        "parser.add_argument(\n",
        "    \"--fp16\",\n",
        "    action=\"store_true\",\n",
        "    help=\"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--fp16_opt_level\",\n",
        "    type=str,\n",
        "    default=\"O1\",\n",
        "    help=\"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3'].\"\n",
        "    \"See details at https://nvidia.github.io/apex/amp.html\",\n",
        ")\n",
        "parser.add_argument(\"--server_ip\", type=str, default=\"\", help=\"Can be used for distant debugging.\")\n",
        "parser.add_argument(\"--server_port\", type=str, default=\"\", help=\"Can be used for distant debugging.\")\n",
        "\n",
        "parser.add_argument(\"--threads\", type=int, default=1, help=\"multiple threads for converting example to features\")\n",
        "args = parser.parse_args(\"\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aEMWxY4q2Yot"
      },
      "source": [
        "## Modifiable args"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IakRJGPu10H5"
      },
      "outputs": [],
      "source": [
        "# modify the arguments as per your requirement\n",
        "\n",
        "args.do_train = True\n",
        "args.do_eval = True\n",
        "args.per_gpu_train_batch_size = 8\n",
        "args.per_gpu_eval_batch_size = 128\n",
        "args.learning_rate = 3e-5\n",
        "args.num_train_epochs = 3\n",
        "args.no_cuda = False\n",
        "args.max_seq_length = 384\n",
        "args.doc_stride = 128\n",
        "args.save_steps = 2\n",
        "args.overwrite_output_dir = True\n",
        "args.data_process_batch = -1\n",
        "args.thread = 64\n",
        "args.verbose_logging = True"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9lG7VeaY2J1z"
      },
      "source": [
        "## main function to do training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VE5HgqSi10H6"
      },
      "outputs": [],
      "source": [
        "\n",
        "def main(args):\n",
        "    \n",
        "    if args.doc_stride >= args.max_seq_length - args.max_query_length:\n",
        "        logger.warning(\n",
        "            \"WARNING - You've set a doc stride which may be superior to the document length in some \"\n",
        "            \"examples. This could result in errors when building features from the examples. Please reduce the doc \"\n",
        "            \"stride or increase the maximum length to ensure the features are correctly built.\"\n",
        "        )\n",
        "\n",
        "    if (\n",
        "        os.path.exists(args.output_dir)\n",
        "        and os.listdir(args.output_dir)\n",
        "        and args.do_train\n",
        "        and not args.overwrite_output_dir\n",
        "    ):\n",
        "        raise ValueError(\n",
        "            \"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.\".format(\n",
        "                args.output_dir\n",
        "            )\n",
        "        )\n",
        "\n",
        "    # Setup distant debugging if needed\n",
        "    if args.server_ip and args.server_port:\n",
        "        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script\n",
        "        import ptvsd\n",
        "\n",
        "        print(\"Waiting for debugger attach\")\n",
        "        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)\n",
        "        ptvsd.wait_for_attach()\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    # Setup CUDA, GPU & distributed training\n",
        "    if args.local_rank == -1 or args.no_cuda:\n",
        "        device = torch.device(\"cuda\" if torch.cuda.is_available() and not args.no_cuda else \"cpu\")\n",
        "        args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()\n",
        "    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs\n",
        "        torch.cuda.set_device(args.local_rank)\n",
        "        device = torch.device(\"cuda\", args.local_rank)\n",
        "        torch.distributed.init_process_group(backend=\"nccl\")\n",
        "        args.n_gpu = 1\n",
        "    args.device = device\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    # Setup logging\n",
        "    logging.basicConfig(\n",
        "        format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
        "        datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "        level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,\n",
        "    )\n",
        "    logger.warning(\n",
        "        \"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s\",\n",
        "        args.local_rank,\n",
        "        device,\n",
        "        args.n_gpu,\n",
        "        bool(args.local_rank != -1),\n",
        "        args.fp16,\n",
        "    )\n",
        "\n",
        "\n",
        "\n",
        "    # Set seed\n",
        "    set_seed(args)\n",
        "\n",
        "    # Load pretrained model and tokenizer\n",
        "    if args.local_rank not in [-1, 0]:\n",
        "        # Make sure only the first process in distributed training will download model & vocab\n",
        "        torch.distributed.barrier()\n",
        "\n",
        "\n",
        "\n",
        "    args.model_type = args.model_type.lower()\n",
        "    config = AutoConfig.from_pretrained(\n",
        "        args.config_name if args.config_name else args.model_name_or_path,\n",
        "        cache_dir=args.cache_dir if args.cache_dir else None,\n",
        "    )\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\n",
        "        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,\n",
        "        do_lower_case=args.do_lower_case,\n",
        "        cache_dir=args.cache_dir if args.cache_dir else None,\n",
        "    )\n",
        "    model = AutoModelForQuestionAnswering.from_pretrained(\n",
        "        args.model_name_or_path,\n",
        "        from_tf=bool(\".ckpt\" in args.model_name_or_path),\n",
        "        config=config,\n",
        "        cache_dir=args.cache_dir if args.cache_dir else None,\n",
        "    )\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    if args.local_rank == 0:\n",
        "        # Make sure only the first process in distributed training will download model & vocab\n",
        "        torch.distributed.barrier()\n",
        "\n",
        "    model.to(args.device)\n",
        "\n",
        "    logger.info(\"Training/evaluation parameters %s\", args)\n",
        "\n",
        "    # Before we do anything with models, we want to ensure that we get fp16 execution of torch.einsum if args.fp16 is set.\n",
        "    # Otherwise it'll default to \"promote\" mode, and we'll get fp32 operations. Note that running `--fp16_opt_level=\"O2\"` will\n",
        "    # remove the need for this code, but it is still valid.\n",
        "    if args.fp16:\n",
        "        try:\n",
        "            import apex\n",
        "\n",
        "            apex.amp.register_half_function(torch, \"einsum\")\n",
        "        except ImportError:\n",
        "            raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to use fp16 training.\")\n",
        "\n",
        "    # Training\n",
        "    if args.do_train:\n",
        "        train_dataset = read_saved_data(\"./cached_train_distilbert-base-uncased_384_dir\", evaluate=False, output_examples=False)\n",
        "        global_step, tr_loss = train(args, train_dataset, model, tokenizer)\n",
        "        logger.info(\" global_step = %s, average loss = %s\", global_step, tr_loss)\n",
        "\n",
        "    # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory\n",
        "    results = {}\n",
        "\n",
        "    return results, model, tokenizer, tr_loss\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ta_Tyenf2MEi"
      },
      "source": [
        "## Run training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yL2cB0wL10H6"
      },
      "outputs": [],
      "source": [
        "results, model, tokenizer = main(args)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pKcoOPISYlPd",
        "outputId": "de45b93f-d07f-4423-e530-9dd8139f8b72"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Final Results on Dev Set after 4 epochs of finetuning is: {'exact': 34.20365535248042, 'f1': 38.922716753576815, 'total': 11873, 'HasAns_exact': 68.45479082321188, 'HasAns_f1': 77.90644669622428, 'HasAns_total': 5928, 'NoAns_exact': 0.050462573591253154, 'NoAns_f1': 0.050462573591253154, 'NoAns_total': 5945, 'best_exact': 50.10528088941295, 'best_exact_thresh': 0.0, 'best_f1': 50.10738650720121, 'best_f1_thresh': 0.0}\n"
          ]
        }
      ],
      "source": [
        "print(\"Final Results on Dev Set after {} epochs of finetuning is:\".format(args.num_train_epochs), results)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluation"
      ],
      "metadata": {
        "id": "zvv9PRah_mKy"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9rJwSON1YmU4"
      },
      "source": [
        "## Separate eval setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4TGO_O1prg6F"
      },
      "source": [
        "Setup CUDA, GPU & distributed training if not done before"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cLdwF7lGrf8P"
      },
      "outputs": [],
      "source": [
        "# Setup CUDA, GPU & distributed training\n",
        "if args.local_rank == -1 or args.no_cuda:\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() and not args.no_cuda else \"cpu\")\n",
        "    args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()\n",
        "else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs\n",
        "    torch.cuda.set_device(args.local_rank)\n",
        "    device = torch.device(\"cuda\", args.local_rank)\n",
        "    torch.distributed.init_process_group(backend=\"nccl\")\n",
        "    args.n_gpu = 1\n",
        "args.device = device"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Eval functions"
      ],
      "metadata": {
        "id": "6zqDzbt0ATD9"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dCa98JpczTIv"
      },
      "outputs": [],
      "source": [
        "def evaluate_for_batch(args, dataset, examples, features, model, tokenizer, prefix=\"\"):\n",
        "\n",
        "    if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:\n",
        "        os.makedirs(args.output_dir)\n",
        "\n",
        "    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)\n",
        "\n",
        "    # Note that DistributedSampler samples randomly\n",
        "    eval_sampler = SequentialSampler(dataset)\n",
        "    eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)\n",
        "\n",
        "    # multi-gpu evaluate\n",
        "    if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):\n",
        "        model = torch.nn.DataParallel(model)\n",
        "\n",
        "    # Eval!\n",
        "    logger.info(\"***** Running evaluation {} *****\".format(prefix))\n",
        "    logger.info(\"  Num examples = %d\", len(dataset))\n",
        "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
        "\n",
        "    all_results = []\n",
        "    start_time = timeit.default_timer()\n",
        "\n",
        "    loss = []\n",
        "\n",
        "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
        "        model.eval()\n",
        "        batch = tuple(t.to(args.device) for t in batch)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            inputs = {\n",
        "                \"input_ids\": batch[0],\n",
        "                \"attention_mask\": batch[1],\n",
        "                \"token_type_ids\": batch[2],\n",
        "            }\n",
        "\n",
        "            if args.model_type in [\"xlm\", \"roberta\", \"distilbert\", \"camembert\", \"bart\"]:\n",
        "                del inputs[\"token_type_ids\"]\n",
        "\n",
        "            feature_indices = batch[3]\n",
        "\n",
        "            start_positions = batch[6]\n",
        "            end_positions = batch[7]\n",
        "\n",
        "            # XLNet and XLM use more arguments for their predictions\n",
        "            if args.model_type in [\"xlnet\", \"xlm\"]:\n",
        "                inputs.update({\"cls_index\": batch[4], \"p_mask\": batch[5]})\n",
        "                # for lang_id-sensitive xlm models\n",
        "                if hasattr(model, \"config\") and hasattr(model.config, \"lang2id\"):\n",
        "                    inputs.update(\n",
        "                        {\"langs\": (torch.ones(batch[0].shape, dtype=torch.int64) * args.lang_id).to(args.device)}\n",
        "                    )\n",
        "            outputs = model(**inputs)\n",
        "            \n",
        "\n",
        "        for i, feature_index in enumerate(feature_indices):\n",
        "            eval_feature = features[feature_index.item()]\n",
        "            unique_id = int(eval_feature.unique_id)\n",
        "\n",
        "            output = [to_list(output[i]) for output in outputs]\n",
        "\n",
        "            # Some models (XLNet, XLM) use 5 arguments for their predictions, while the other \"simpler\"\n",
        "            # models only use two.\n",
        "            if len(output) >= 5:\n",
        "                start_logits = output[0]\n",
        "                start_top_index = output[1]\n",
        "                end_logits = output[2]\n",
        "                end_top_index = output[3]\n",
        "                cls_logits = output[4]\n",
        "\n",
        "                result = SquadResult(\n",
        "                    unique_id,\n",
        "                    start_logits,\n",
        "                    end_logits,\n",
        "                    start_top_index=start_top_index,\n",
        "                    end_top_index=end_top_index,\n",
        "                    cls_logits=cls_logits,\n",
        "                )\n",
        "\n",
        "            else:\n",
        "                start_logits, end_logits = output\n",
        "                result = SquadResult(unique_id, start_logits, end_logits)\n",
        "\n",
        "                #EXPERIMENTAL - GET LOSS\n",
        "\n",
        "                seq_length = len(eval_feature.input_ids)\n",
        "\n",
        "                def compute_loss(logits, positions):\n",
        "                  one_hot_positions = tf.one_hot(\n",
        "                      positions, depth = seq_length, dtype = tf.float32)\n",
        "                  log_probs = tf.nn.log_softmax(logits, axis = -1)\n",
        "                  loss = -tf.reduce_mean(\n",
        "                      tf.reduce_sum(one_hot_positions * log_probs, axis=-1))\n",
        "                  return loss\n",
        "\n",
        "                start_loss = compute_loss(start_logits, start_positions.cpu().data.numpy())\n",
        "                end_loss = compute_loss(end_logits, end_positions.cpu().data.numpy())\n",
        "\n",
        "                total_loss = (start_loss + end_loss) / 2.0\n",
        "\n",
        "                loss.append(total_loss)\n",
        "\n",
        "                #/EXPERIMENTAL\n",
        "\n",
        "            all_results.append(result)\n",
        "\n",
        "    evalTime = timeit.default_timer() - start_time\n",
        "    logger.info(\"  Evaluation done in total %f secs (%f sec per example)\", evalTime, evalTime / len(dataset))\n",
        "\n",
        "    # Compute predictions\n",
        "    output_prediction_file = os.path.join(args.output_dir, \"predictions_{}.json\".format(prefix))\n",
        "    output_nbest_file = os.path.join(args.output_dir, \"nbest_predictions_{}.json\".format(prefix))\n",
        "\n",
        "    if args.version_2_with_negative:\n",
        "        output_null_log_odds_file = os.path.join(args.output_dir, \"null_odds_{}.json\".format(prefix))\n",
        "    else:\n",
        "        output_null_log_odds_file = None\n",
        "\n",
        "    # XLNet and XLM use a more complex post-processing procedure\n",
        "    if args.model_type in [\"xlnet\", \"xlm\"]:\n",
        "        start_n_top = model.config.start_n_top if hasattr(model, \"config\") else model.module.config.start_n_top\n",
        "        end_n_top = model.config.end_n_top if hasattr(model, \"config\") else model.module.config.end_n_top\n",
        "\n",
        "        predictions = compute_predictions_log_probs(\n",
        "            examples,\n",
        "            features,\n",
        "            all_results,\n",
        "            args.n_best_size,\n",
        "            args.max_answer_length,\n",
        "            output_prediction_file,\n",
        "            output_nbest_file,\n",
        "            output_null_log_odds_file,\n",
        "            start_n_top,\n",
        "            end_n_top,\n",
        "            args.version_2_with_negative,\n",
        "            tokenizer,\n",
        "            args.verbose_logging,\n",
        "        )\n",
        "    else:\n",
        "        predictions = compute_predictions_logits(\n",
        "            examples,\n",
        "            features,\n",
        "            all_results,\n",
        "            args.n_best_size,\n",
        "            args.max_answer_length,\n",
        "            args.do_lower_case,\n",
        "            output_prediction_file,\n",
        "            output_nbest_file,\n",
        "            output_null_log_odds_file,\n",
        "            args.verbose_logging,\n",
        "            args.version_2_with_negative,\n",
        "            args.null_score_diff_threshold,\n",
        "            tokenizer,\n",
        "        )\n",
        "\n",
        "    # Compute the F1 and exact scores.\n",
        "    results = squad_evaluate(examples, predictions)\n",
        "    return results, loss\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D3yXKOhv5o6I"
      },
      "outputs": [],
      "source": [
        "from collections import OrderedDict\n",
        "def state_dict_sum(sd_0, sd_1):\n",
        "  sd = OrderedDict()\n",
        "  for key in sd_0.keys():\n",
        "    sd[key] = sd_0[key] + sd_1[key]\n",
        "  return sd\n",
        "\n",
        "def state_dict_sum_coef(sd_0, sd_1, alpha):\n",
        "  sd = OrderedDict()\n",
        "  for key in sd_0.keys():\n",
        "    sd[key] = sd_0[key] + alpha * sd_1[key]\n",
        "  return sd\n",
        "\n",
        "def state_dict_mult_scalar(sd_0, sd_1):\n",
        "  result = 0\n",
        "  keys = list(sd_0.keys())\n",
        "  for i in tqdm(range(len(keys))):\n",
        "    result += torch.sum(\n",
        "        torch.mul(sd_0[keys[i]], sd_1[keys[i]])\n",
        "        ).item() \n",
        "\n",
        "  return result\n",
        "\n",
        "def state_dict_mult_n(sd_0, alpha):\n",
        "  sd = OrderedDict()\n",
        "  for key in sd_0.keys():\n",
        "    sd[key] = alpha * sd_0[key]\n",
        "  return sd\n",
        "\n",
        "def state_dict_diff(sd_0, sd_1):\n",
        "  sd = OrderedDict()\n",
        "  for key in sd_0.keys():\n",
        "    sd[key] = sd_1[key] - sd_0[key]\n",
        "  return sd"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1D Linear Interpolation"
      ],
      "metadata": {
        "id": "ESKetF8z_ojh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"./drive/My Drive/distilbert-squad-100/checkpoint-48000\")\n",
        "dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True)\n",
        "dataset1, examples1, features1 = load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-wrXPv_yE1wV",
        "outputId": "e7ffbee7-4fab-4fc8-e757-d9a02f2c9737"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 9/9 [00:00<00:00,  9.28it/s]\n",
            "convert squad examples to features:   0%|          | 0/1849 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:1423: FutureWarning: The `max_len` attribute has been deprecated and will be removed in a future version, use `model_max_length` instead.\n",
            "  FutureWarning,\n",
            "convert squad examples to features: 100%|██████████| 1849/1849 [00:16<00:00, 115.21it/s]\n",
            "add example index and unique id: 100%|██████████| 1849/1849 [00:00<00:00, 625677.14it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WH6eZs2WEyIu"
      },
      "outputs": [],
      "source": [
        "def run_1D_interpolation(args, model_path_0, model_path_1, tokenizer_path, n, alpha_range):\n",
        "\n",
        "  model_0 = AutoModelForQuestionAnswering.from_pretrained(model_path_0) \n",
        "  model_1 = AutoModelForQuestionAnswering.from_pretrained(model_path_1) \n",
        "  tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)\n",
        "\n",
        "  #caching d/e/f, will skip if already done\n",
        "  dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True)\n",
        "  dataset_t, examples_t, features_t = load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=True)\n",
        "\n",
        "  start_pos = [x.start_position for x in features_t]\n",
        "  end_pos = [x.end_position for x in features_t]\n",
        "\n",
        "  dataset_mod = TensorDataset()\n",
        "  dataset_mod.tensors = tuple([dataset.tensors[0],\n",
        "                              dataset.tensors[1],\n",
        "                              dataset.tensors[2],\n",
        "                              dataset.tensors[3],\n",
        "                              dataset.tensors[4],\n",
        "                              dataset.tensors[5],\n",
        "                              start_pos, \n",
        "                              end_pos])\n",
        "\n",
        "\n",
        "  sd_0 = model_0.state_dict()\n",
        "  sd_1 = model_1.state_dict()\n",
        "\n",
        "  delta = state_dict_diff(sd_0, sd_1)\n",
        "  x = [alpha_range[0] + (alpha_range[1] - alpha_range[0]) * i / (n - 1.0) for i in range(0, n)]\n",
        "  y = []\n",
        "  loss = []\n",
        "\n",
        "\n",
        "  for i in tqdm(range(0, n)):\n",
        "    global_step = \"1D_ip_\" + str(i)\n",
        "\n",
        "    sd_n = state_dict_sum_coef(sd_0, delta, x[i])\n",
        "    model_n = AutoModelForQuestionAnswering.from_pretrained(model_path_0, state_dict = sd_n)\n",
        "    model_n.to(args.device)\n",
        "\n",
        "    # Evaluate\n",
        "    result, loss_n = evaluate_for_batch(args, dataset_mod, examples, features, model_n, tokenizer, prefix=global_step)\n",
        "    result = dict((k + (\"_{}\".format(global_step) if global_step else \"\"), v) for k, v in result.items())\n",
        "    y.append(result)\n",
        "    loss.append(loss_n)\n",
        "\n",
        "  return x, y, loss"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run 1D Linear interpolation"
      ],
      "metadata": {
        "id": "6uwAoTcjABL7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_path_0 = \"./drive/My Drive/distilbert-init/checkpoint-init\"\n",
        "model_path_1 = \"./drive/My Drive/distilbert-squad-100/checkpoint-48000\"\n",
        "tokenizer_path = \"./drive/My Drive/distilbert-squad-100/checkpoint-48000\"\n",
        "\n",
        "x, y, od_loss = run_1D_interpolation(args,\n",
        "                                     model_path_0, model_path_1, tokenizer_path,\n",
        "                                     n=2, alpha_range = [-0.5, 1.5])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e2wWYOW9ziLm",
        "outputId": "6b6db9fb-824c-4a17-8188-94dd8e169349"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  0%|          | 0/2 [00:00<?, ?it/s]\n",
            "Evaluating:   0%|          | 0/15 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   7%|▋         | 1/15 [00:00<00:13,  1.00it/s]\u001b[A\n",
            "Evaluating:  13%|█▎        | 2/15 [00:01<00:12,  1.02it/s]\u001b[A\n",
            "Evaluating:  20%|██        | 3/15 [00:02<00:11,  1.02it/s]\u001b[A\n",
            "Evaluating:  27%|██▋       | 4/15 [00:03<00:10,  1.02it/s]\u001b[A\n",
            "Evaluating:  33%|███▎      | 5/15 [00:04<00:09,  1.03it/s]\u001b[A\n",
            "Evaluating:  40%|████      | 6/15 [00:05<00:08,  1.03it/s]\u001b[A\n",
            "Evaluating:  47%|████▋     | 7/15 [00:06<00:07,  1.03it/s]\u001b[A\n",
            "Evaluating:  53%|█████▎    | 8/15 [00:07<00:06,  1.03it/s]\u001b[A\n",
            "Evaluating:  60%|██████    | 9/15 [00:08<00:05,  1.04it/s]\u001b[A\n",
            "Evaluating:  67%|██████▋   | 10/15 [00:09<00:04,  1.03it/s]\u001b[A\n",
            "Evaluating:  73%|███████▎  | 11/15 [00:10<00:03,  1.03it/s]\u001b[A\n",
            "Evaluating:  80%|████████  | 12/15 [00:11<00:02,  1.03it/s]\u001b[A\n",
            "Evaluating:  87%|████████▋ | 13/15 [00:12<00:01,  1.03it/s]\u001b[A\n",
            "Evaluating:  93%|█████████▎| 14/15 [00:13<00:00,  1.04it/s]\u001b[A\n",
            "Evaluating: 100%|██████████| 15/15 [00:14<00:00,  1.05it/s]\n",
            " 50%|█████     | 1/2 [00:19<00:19, 19.95s/it]\n",
            "Evaluating:   0%|          | 0/15 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   7%|▋         | 1/15 [00:00<00:13,  1.01it/s]\u001b[A\n",
            "Evaluating:  13%|█▎        | 2/15 [00:02<00:16,  1.28s/it]\u001b[A\n",
            "Evaluating:  20%|██        | 3/15 [00:03<00:13,  1.14s/it]\u001b[A\n",
            "Evaluating:  27%|██▋       | 4/15 [00:04<00:11,  1.07s/it]\u001b[A\n",
            "Evaluating:  33%|███▎      | 5/15 [00:05<00:10,  1.03s/it]\u001b[A\n",
            "Evaluating:  40%|████      | 6/15 [00:06<00:09,  1.01s/it]\u001b[A\n",
            "Evaluating:  47%|████▋     | 7/15 [00:07<00:08,  1.00s/it]\u001b[A\n",
            "Evaluating:  53%|█████▎    | 8/15 [00:08<00:06,  1.01it/s]\u001b[A\n",
            "Evaluating:  60%|██████    | 9/15 [00:09<00:05,  1.02it/s]\u001b[A\n",
            "Evaluating:  67%|██████▋   | 10/15 [00:10<00:04,  1.02it/s]\u001b[A\n",
            "Evaluating:  73%|███████▎  | 11/15 [00:11<00:03,  1.02it/s]\u001b[A\n",
            "Evaluating:  80%|████████  | 12/15 [00:12<00:02,  1.02it/s]\u001b[A\n",
            "Evaluating:  87%|████████▋ | 13/15 [00:13<00:01,  1.02it/s]\u001b[A\n",
            "Evaluating:  93%|█████████▎| 14/15 [00:14<00:00,  1.02it/s]\u001b[A\n",
            "Evaluating: 100%|██████████| 15/15 [00:14<00:00,  1.01it/s]\n",
            "100%|██████████| 2/2 [00:44<00:00, 22.25s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "od_loss"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XBUnv4mMfawt",
        "outputId": "1f47fb4b-689d-4f45-fd6b-66f13bd9b9bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[<tf.Tensor: shape=(), dtype=float32, numpy=6.020652>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.014652>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.007058>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.010991>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0198617>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.024008>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029004>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029711>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0229034>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.021759>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.047306>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.049079>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0501122>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.058899>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.044916>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.072623>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.087635>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.060817>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0877943>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0832124>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0591393>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0646806>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.055711>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0642986>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0645666>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.966044>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9760427>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9672627>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.974604>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9712696>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9582505>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.958657>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.949139>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.963865>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.984406>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0453963>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0545077>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.044544>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0496635>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.047165>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029519>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0155883>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.035208>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0364714>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0273676>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.04771>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0599556>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0406084>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.041291>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029259>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0324364>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.021096>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.03209>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.021356>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0197496>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029624>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0338416>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.01767>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0192018>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.03665>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0352077>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.037456>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.057578>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0764894>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0852103>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.08667>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.114327>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.078472>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.032686>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.063458>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034859>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038254>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.013953>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0320425>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0284576>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0475225>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0310802>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0376577>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046273>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0450063>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.062147>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.047127>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0371327>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.006127>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9941983>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.006127>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.006748>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0022173>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0065427>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0062027>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.068514>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.041619>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0350356>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0664587>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0388327>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.033701>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0344815>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.030024>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029073>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.043724>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9985003>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.009403>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.996955>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9975634>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0123057>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0455017>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.043>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0450363>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038336>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0413575>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0364637>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.037676>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0345>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0483065>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0052395>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9927173>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9970145>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.000682>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.003516>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0492682>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.050828>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0471296>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0447655>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029815>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0118866>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.008242>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0134497>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.011489>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.043632>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0426607>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0240145>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0328417>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029024>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.032832>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0696774>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0468836>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.048301>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0641837>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0586934>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0071664>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0079603>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0013895>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.004681>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.022368>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9840245>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.007433>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9997654>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0086727>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.018882>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.027185>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.066>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.042939>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0486507>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.06464>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.04679>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0345263>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.043991>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0236516>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0409613>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.992258>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9838963>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.963619>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.984422>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9732466>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9948254>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.95477>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.019724>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9721355>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.994893>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.023851>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0398245>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0509233>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0715175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0433235>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0507197>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0682077>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0361614>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0333433>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0430007>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.043481>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.079239>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0429125>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052004>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0382185>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.025661>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0401907>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0425916>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0564632>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.028861>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0348344>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0253963>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.045536>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0447197>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0333576>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.013883>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.009533>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.014991>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0049257>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.019966>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0149107>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0113764>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0056686>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.011183>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.014104>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.007723>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.027132>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.026722>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0301476>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.033556>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.991642>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.991912>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0115533>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0269012>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.013076>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0349627>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0412703>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046661>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046632>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.037721>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9786215>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9601088>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9970126>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9974365>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9785633>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0438375>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.006711>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0241356>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.026065>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.031993>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.027368>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0133505>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0273123>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.012971>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0118217>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.035654>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.037554>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0575047>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052493>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0367966>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0405445>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.022519>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0365167>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0435133>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.043873>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.025111>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.021056>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0346727>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.022045>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038024>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0249567>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.010168>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0262814>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0256257>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0086374>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.004875>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.993567>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.026856>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.005541>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0073643>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0633>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0788054>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0742483>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0540123>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.073341>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0054398>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9791756>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0173054>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0000887>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.017499>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.025071>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.014379>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0110106>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0191464>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.013508>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.060899>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0549154>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.053737>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.014571>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.059284>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.024287>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0361166>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0387144>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9968357>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0507298>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0426073>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052724>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0260153>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0140057>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0077677>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9848228>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9990215>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.977708>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9786444>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9980793>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.096971>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.092964>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0953627>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0947065>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0959587>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0372944>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0287766>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.027334>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0391393>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0234766>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0199904>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.028382>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0257835>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0296316>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029687>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.060525>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0497017>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052571>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.059343>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.073942>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.03419>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0548306>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.036805>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0316963>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.032001>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9974127>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0241685>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.965294>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.992344>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9697647>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0519876>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.065767>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0673943>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.058871>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0554867>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0232954>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0197735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0236254>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.018356>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0194435>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0832148>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0853834>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052515>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0436325>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.07076>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0258613>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0067167>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0256352>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.00768>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.022853>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0648174>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034009>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0481257>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0593147>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0424356>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0336037>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.049616>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.041061>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0513086>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0668516>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0980244>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.088381>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0562525>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0930915>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.070598>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0470085>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.02376>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0379753>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0221844>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0485>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046158>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.060487>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0601826>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.072242>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0584064>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.03281>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.062173>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.055134>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0628467>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.075898>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.03704>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0681734>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.054235>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038659>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0390015>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0319743>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.050208>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0615463>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.079584>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.079344>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034629>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.027688>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9984264>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.035759>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0137844>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.050205>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.055494>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0983605>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0697346>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.056004>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.01521>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0165253>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.016449>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0183773>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0255632>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0426707>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.051225>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.037162>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0543904>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0099645>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0277705>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046749>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0472903>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0244284>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0489645>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0460234>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0451384>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0527244>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.048057>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046959>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.082>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0824842>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0816946>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.082211>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.053605>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.041657>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0494156>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0406904>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0414176>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0528007>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0728703>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.072564>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.075531>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0776987>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0528555>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.05621>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.040184>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.058689>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0503073>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0606585>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.053175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0518637>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.044937>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0296607>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0655594>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0667005>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0632887>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.055435>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.063957>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.033529>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.036458>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034939>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.027626>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.090849>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.072638>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0700884>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.06226>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0546203>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.056922>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0678163>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046744>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.063814>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0518885>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.037592>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038047>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0156517>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.02254>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.022993>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.026163>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0530505>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0449133>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0529146>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0156183>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.01058>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0496507>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0662413>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0671883>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0852766>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.086454>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.054476>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0580845>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0626297>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.062872>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0551767>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0536256>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.057273>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0409713>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0759087>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.110954>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0849624>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.061842>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.094816>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.093656>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.075816>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.040656>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0375004>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.033415>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0764985>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1145477>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.138064>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.079875>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1129923>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1076937>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0813484>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0362134>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0783434>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.077859>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.064928>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0710783>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0460978>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0503855>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.043794>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0398207>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0349617>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.040357>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.035774>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046241>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0373564>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.073308>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.049983>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0775075>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0332317>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0873127>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0721025>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0718317>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.090705>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0814915>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0313663>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0681543>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.05157>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0144925>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.030527>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.062853>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.014262>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0662246>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0202456>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.071685>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.037555>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0401816>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0147943>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.021122>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0375338>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0227623>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.02765>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0236692>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.049235>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0439644>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.049735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.063587>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0563493>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.054879>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0432897>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0458984>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0445404>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0426226>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0298343>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0450993>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0270133>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034055>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0501513>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0622826>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0324917>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.054986>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0802436>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.053085>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.051859>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.054549>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.033641>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038927>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0852737>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.074493>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0587244>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0831985>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0857334>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0578976>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.060829>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.047611>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.072089>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0738707>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.050288>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0740385>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0520906>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0618258>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.057211>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0508537>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038889>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.036767>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0371933>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.000192>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.018442>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0300207>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.024047>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.016611>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0132732>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0542026>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.059681>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.060312>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0650835>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.047066>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.063284>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.027027>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0361886>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.018403>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0231943>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.024742>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.023475>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.997904>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0343394>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.000308>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0573444>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.047142>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0080633>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.025442>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.068674>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.068844>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0834756>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0810223>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0747604>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0420275>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.048105>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.049943>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0613976>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0326023>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0280685>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0000286>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.042575>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0360518>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.024051>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0962343>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0924106>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0973015>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.091871>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0706797>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0425916>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.091819>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0574293>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.075851>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046569>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.055889>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.070469>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0523825>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.058192>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.064881>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.093009>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.097625>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0787163>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.07824>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0657043>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.065185>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.041686>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.032213>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0827885>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0630474>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0359135>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0548263>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038454>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0421557>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.048128>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0599766>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.047577>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0557127>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0741334>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.106928>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.106928>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.093918>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.051719>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0560446>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0458517>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0527>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.069636>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.050041>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0506573>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0521154>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0555696>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0642533>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0596886>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0577745>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0425587>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0251412>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.07477>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.071613>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.084571>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0545344>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0545344>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0182295>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0182295>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.079883>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0615387>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0840845>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0306425>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.032673>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.02834>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.009897>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0455003>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.045328>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0574484>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0646358>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052572>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0698795>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.037185>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.035677>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.040624>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0221443>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.054303>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.073039>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0447464>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0569086>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0669355>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0433855>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0599604>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0433855>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0292134>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.995036>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9931335>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9865236>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.999615>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0561957>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.030445>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.040962>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0492353>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.010421>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.029253>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0379987>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0702477>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0834703>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0402308>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0347767>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0589223>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0531526>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0723443>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0997133>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1007996>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0864043>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.056033>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0886536>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0785313>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0256743>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0322175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0297384>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0286827>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0414534>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.055026>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0743027>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.061208>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.070016>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034893>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0350018>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0299706>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.055716>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.059515>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0549603>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.032728>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0169063>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0523033>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.064025>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.041942>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.056226>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0810113>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0978107>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.089979>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.083827>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.083412>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.050486>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0227227>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.044038>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0241594>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0256677>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.015927>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0608444>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0614376>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.071765>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0741105>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0686135>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.018118>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0579195>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.018427>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0545874>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0345364>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046938>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0559316>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.057369>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0445147>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.058976>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.071946>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.070633>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.086876>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0420995>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.041214>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0413094>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038187>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.050564>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.09164>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0847425>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.087677>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.035652>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.056598>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.070593>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.071145>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0667124>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0568175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0591135>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.050842>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0622835>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0457687>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.066084>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0526285>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0505667>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0515857>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0164595>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.049697>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.039523>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0508857>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9838605>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9981546>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.984621>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.99891>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.993938>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.007032>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9818764>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9986963>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.065527>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0583954>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.083475>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.056127>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.044023>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.044217>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0476317>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.030793>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.031278>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.030859>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.031045>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0758233>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.112306>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0706863>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0718517>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0891066>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0832815>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0537796>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.053336>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052805>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0523252>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.030875>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.03051>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0654125>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.04245>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0343657>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.039036>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.046448>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.076044>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0620184>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.080776>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.063916>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.059232>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0422764>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.030491>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0323696>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0241766>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0316606>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0752115>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.06056>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.058843>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.056628>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.058402>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.04348>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0534463>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.055448>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.036487>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0554075>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.069273>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0559483>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.080437>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.073845>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.074637>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0517817>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0526295>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.087295>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0500646>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.104475>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0811963>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.070092>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.083799>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.088357>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034855>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0370007>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0424337>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0302567>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0274057>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.999738>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.006014>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.994685>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0032954>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0217915>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052207>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.016863>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.058388>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0437965>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.017966>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034442>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0249305>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.017672>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0396147>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0139627>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0096526>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.018628>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0426455>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0139704>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0786924>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.075171>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.068679>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0775247>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.109062>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.038259>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0735044>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0668173>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.040882>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0781126>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.078998>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.052305>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0398006>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.084426>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0578017>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0427456>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9930954>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.018852>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.028782>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0531054>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0877767>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1132927>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1091337>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.09599>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.109072>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0640326>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.053891>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0867033>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.047843>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.053542>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.106904>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0844955>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0738845>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0891175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1024294>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0872307>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.090557>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0853057>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0843854>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.092719>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.031735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.034382>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.01515>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.141631>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0883856>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.087965>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0999556>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1153984>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0694866>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.076925>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0785336>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.090452>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0672655>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.118222>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.144846>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.105406>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.074954>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1303377>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0841746>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0911045>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0836253>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.05118>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.071494>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0787983>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.081006>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.073722>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.07938>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.086707>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.120784>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.081362>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1476803>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.167291>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1829386>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1809416>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1661434>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0862193>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.116395>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1156125>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1540537>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.124459>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1556697>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1608763>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.130624>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.2016325>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.154302>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1782827>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.140584>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.141312>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1283007>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.146948>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.133623>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1399393>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.10275>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0932007>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.093114>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1033936>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.061609>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0865736>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0856314>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.971063>,\n",
              "  ...],\n",
              " [<tf.Tensor: shape=(), dtype=float32, numpy=12.963846>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.295995>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.890839>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.095671>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.486456>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.761435>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.580039>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.457152>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.449137>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.8906>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.724556>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.286415>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.423452>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.554419>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.900426>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.342993>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.107311>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.986517>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.899956>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.838854>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.5592694>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.294317>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.0160885>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.110559>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.022057>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.321736>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0261507>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.477772>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.2582397>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.227848>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.658993>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.625729>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.984274>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.928444>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.148457>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.697844>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.751521>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.325346>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.005705>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.226444>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.688213>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.86594>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.536177>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.369175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.2303505>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.98237>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.13735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.54188>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.634861>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.987103>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.972847>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.681782>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.37059>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.913792>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.956581>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.265844>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.613506>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.256195>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.763374>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.517494>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.881138>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.933697>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.680803>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.778423>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.730959>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.019669>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.248201>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.056929>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.6676445>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.46794>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.694035>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.839544>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.705708>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.185806>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.289859>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.158649>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.222681>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.565921>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.392971>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.656656>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.527555>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.146557>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.303825>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.025223>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.83828>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.025223>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.463978>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.022585>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.754723>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.470303>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.484793>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.547167>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.332556>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.35683>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.724764>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.356539>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.859446>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.186703>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.200669>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.3705854>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.162186>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.542027>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.557137>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.888937>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.942657>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.202463>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.830042>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.31986>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.593468>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.082485>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.504562>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.090884>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.769833>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.407845>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.107541>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.374947>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.612937>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.86077>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.829774>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.393467>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.850559>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.562096>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.958202>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.376585>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.83443>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.807483>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.721262>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.584166>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.001116>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.763977>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.622721>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.728874>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.098587>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.678656>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.039959>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.06898>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.787254>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.69666>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.501072>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.94763>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.419937>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.798771>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.070564>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.104549>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.690178>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.649704>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=22.52391>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.220917>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.330271>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.294346>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.469303>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.743893>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.677963>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.827675>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.7035127>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.53128>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.517628>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.784069>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.238344>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.080711>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.36745>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.186016>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.276241>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.799717>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.71334>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.62328>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.555645>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.289032>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.13469>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.142701>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.408445>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.429152>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.751681>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.077896>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.034443>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.621881>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.757587>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.693874>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.757452>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.357666>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.943953>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.471231>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.343895>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.586315>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.111237>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.141172>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.401108>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.79621>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.50901>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.846642>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.297052>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.593222>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.202341>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.336319>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.264514>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.633862>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.966848>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.164307>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.957492>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.5707855>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.05073>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.912006>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.449554>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.808037>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.10684>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.239076>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.5265255>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.615216>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.365822>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.433105>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.374836>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.00082>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.015018>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.64177>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.85342>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.450027>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.13416>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.798588>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.199106>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.507568>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.986544>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.015743>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.420063>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.651718>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.729534>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.127695>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.875135>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.999761>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.58167>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.692159>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.001982>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.116631>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.972069>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.261183>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.50446>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.611652>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.745707>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.578205>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.312706>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.190556>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.23877>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.563238>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.77985>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.8830585>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.873425>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.805895>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.633317>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.282009>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.937234>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.631424>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.929398>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=22.33416>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.3149395>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.220682>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.956568>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.383045>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.846306>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.095911>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.221542>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.016264>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.871923>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.161007>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.551569>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.472973>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.8452306>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.94867>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.282928>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.687943>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.427582>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.979637>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.615475>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.770055>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.717285>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.190809>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.796135>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.794477>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.428623>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.593534>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.216444>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.596157>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.56897>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.262781>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.027515>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.247536>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.35802>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.747368>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.317154>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.82522>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.450943>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.125458>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.216894>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.624174>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.038245>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.692379>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.150188>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.204994>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.252869>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.37068>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.474974>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.704324>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.990801>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.944494>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.673448>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.256801>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.618862>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.8235536>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9058228>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.409893>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.323595>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.343243>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.122466>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.975231>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.913584>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.59071>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.166428>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.115803>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.220622>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.124088>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.4098463>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.766281>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.474225>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.657914>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.201483>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.573595>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.113249>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.3629265>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.013358>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.958952>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.032427>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.683491>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9097395>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.361769>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.18693>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.341597>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.084479>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.77085>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.39246>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.266857>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.533813>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0459876>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.297045>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.036762>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.146091>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.180716>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.241655>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.942775>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1562624>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.836615>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.577534>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.351259>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.1118464>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.994296>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.412774>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.513079>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.229292>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.066252>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.404133>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.100376>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.779392>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.4744>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.517628>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.362539>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0834208>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.698816>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.520116>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.386744>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.764858>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.873417>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.326416>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.750595>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.012061>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.545063>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.653638>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.39252>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.600512>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.905958>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.936104>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.363466>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.893421>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.635212>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.287594>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.229393>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.113676>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.052904>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.44374>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.544842>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0875125>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=22.33416>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.10342>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.873121>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.605553>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.81852>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.928667>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.626569>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.623209>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.523723>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.214842>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.69059>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.525074>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.94122>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.036636>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.449692>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.52356>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.9149>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.509283>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.846218>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.415106>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.719032>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.413006>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.277618>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.51495>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.490557>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.916204>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.693624>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.337234>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.178661>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.833968>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.485394>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.173788>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.724682>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.556803>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.551102>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.713938>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.67416>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.859079>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.987446>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.919076>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1184397>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.4442625>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.752867>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.950052>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.699917>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.555433>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.823732>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.256687>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.09623>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.026062>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.429281>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.386558>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.5625324>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.262961>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.656347>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.072066>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.845924>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.508583>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.429472>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.928009>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.966759>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.80452>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.909603>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.08302>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.344898>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.955591>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.55352>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.710033>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.508264>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.278903>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.927774>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.295721>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.113636>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.07167>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.977491>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.614155>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.923109>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.1668>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.288433>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.627894>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.182644>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.36479>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.0240097>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.52996>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.41938>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.971367>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.687708>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.905758>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.053278>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.147959>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.152208>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.933163>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.327222>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.786549>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.861687>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.795214>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.18013>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.311298>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.924698>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.460192>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.548649>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.828289>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.639832>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.922514>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.23861>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.62487>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.913548>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.063599>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.89615>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.291735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.811724>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.031946>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.737871>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1540966>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.73857>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.207443>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.164698>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.705493>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.873316>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.675644>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.136976>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.163355>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.051012>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.36701>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.491814>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.336502>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.325773>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.241735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.639265>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.116016>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.53754>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.318453>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.908942>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.04169>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.37748>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.767412>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.115894>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.505613>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.17338>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.865186>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.870682>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.950777>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.226692>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.96276>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.72076>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.591913>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.37721>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.463074>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.515015>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.201753>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.4631195>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.903672>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.634514>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.747738>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.355814>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.723328>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.197048>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.377113>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.316692>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.630268>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.118479>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.27333>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.721544>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.162527>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.4929013>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.501053>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.875052>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.212626>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.23523>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.376206>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.801471>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.440334>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.254595>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.896965>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.893723>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.778194>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.008596>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.421097>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.657812>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.792059>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.66919>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.13245>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.723137>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.580925>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.270108>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.196844>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.454197>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.0087185>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.85351>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.214638>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.904071>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.549442>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.424232>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.193305>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.287281>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.453756>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.79726>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.247253>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.674477>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.88411>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.524323>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.166977>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.107189>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.372673>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.089115>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.510181>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.093666>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.227936>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.139095>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.907318>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.899609>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.350754>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.192638>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.526611>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.034885>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.935257>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.225094>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.941431>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.760881>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.45966>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.318863>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.566782>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.470078>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.543801>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.149397>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.461012>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.919439>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.360278>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.259247>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.591241>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.623974>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.019506>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.637688>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.055471>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.911928>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.480314>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.573414>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.417058>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.235098>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.492043>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.882923>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.439507>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.031551>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.601044>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.846342>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.9145565>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.64796>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.538055>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.798932>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.486792>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.158503>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.381168>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.9342546>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.347343>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.424644>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.171787>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.480633>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.060383>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.739769>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.734468>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.79449>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.624478>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.831856>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.3771305>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.171036>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.161298>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.10651>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.023182>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.789137>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.062193>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.062193>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.853641>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.007772>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.942618>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.7496>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.059275>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.752263>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.845623>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.498497>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.034775>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.36693>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.423714>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.955056>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.060933>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.234718>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.853161>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.91832>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.835659>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.172075>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.855499>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.855499>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.2627735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.2627735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.502674>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.536629>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.635044>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.060709>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.069649>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.871035>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.79524>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.069288>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.474186>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.377905>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.629395>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.774996>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.807418>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.811105>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.073143>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.950945>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.224332>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.73072>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.089993>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.4834175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.000969>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.144058>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.877987>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.763357>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.877987>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.774628>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.264163>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.259157>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.660131>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.777786>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.026326>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.000408>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.34517>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.345306>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.177202>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.92288>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.620947>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.201048>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.90652>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.52302>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.567038>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.555578>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.988762>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.812582>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.313942>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.739252>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.444273>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.782803>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.776203>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.49704>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.996248>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.809082>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.5734215>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.6155205>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.354954>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.068703>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.140675>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.211624>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.067242>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.7902775>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.159981>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.317247>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.00923>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.960249>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.719376>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.373431>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.017972>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.893275>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.3653>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.569534>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.426649>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.792751>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.397099>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.370073>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.631694>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.783318>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.948578>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.865331>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.101837>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.578995>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.437998>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.553694>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.684565>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.062687>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.282508>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.188413>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.657486>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.874888>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.629375>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.964487>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.6712>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.222837>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.010429>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.410564>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.607238>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.405366>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.1007>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.43154>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.56168>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.565496>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.006287>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.755585>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.672383>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.79339>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.108593>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.309223>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.15979>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.22681>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.775034>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.071354>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.638927>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.455332>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.608274>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.645851>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.988897>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.292755>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.124966>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.006462>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.638851>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.663864>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.593544>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.588539>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.727158>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.336506>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.51542>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.690903>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.247261>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.285713>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.354504>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.42536>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.666331>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.248202>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.06929>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.523053>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.421684>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.870433>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.31616>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.2246304>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.286512>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.300503>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.422427>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.035032>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.830494>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.045296>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.685913>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.086342>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.460999>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.321726>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.371548>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.202303>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.352049>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.731941>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.804138>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.646358>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.129665>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.337058>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.471684>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.3769>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.157207>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.196798>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.8378353>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.883131>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.623396>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.949601>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.959568>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.796079>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.573591>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.794926>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.967312>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.935937>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.06747>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.29968>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.920182>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.483749>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.23036>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.291714>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.067457>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.725735>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.548609>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.66734>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.164389>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.890993>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.912334>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.034374>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.940835>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.58345>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.290649>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.6616354>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.1332455>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.7348876>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.019876>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.892376>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.791485>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.608202>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.390114>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.778072>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.83058>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=20.665005>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.233929>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.109346>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.227917>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.249474>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.329473>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.9298525>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.848494>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.635681>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.841648>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.829899>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.996107>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.215534>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.296204>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.731808>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.276714>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.676174>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.616682>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.962812>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.017708>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.111372>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.424849>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.064564>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.74476>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.475159>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.41948>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.156378>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.418237>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.368066>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.375135>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.695461>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.80608>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.998083>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.3611145>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.6608114>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.992849>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.939274>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.138902>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.129129>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.401175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.492411>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.967897>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.752747>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.953598>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.472756>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.382639>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.33106>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.529247>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.5967903>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.5201178>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.8958063>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.222897>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.176135>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.095095>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.665209>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.41534>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.844749>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.025303>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.14344>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.380499>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.345749>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.496414>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.367501>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9711723>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.900777>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.78989>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.915234>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.413816>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.811767>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.420043>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.637836>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.992447>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.095187>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.0909>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.559113>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.368465>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.052406>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.9250965>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.109378>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.279594>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=19.615685>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.274326>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.173066>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.178484>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.519905>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.416994>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.9020104>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=6.9172363>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.240936>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.418032>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.3310175>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=8.795068>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.92137>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.7693243>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.192781>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.911339>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.097032>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.665928>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.690584>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.091273>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=9.92581>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=17.791508>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.091812>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.620541>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.0440445>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.40966>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.764133>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.616067>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.961475>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=15.671786>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=13.654804>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=5.7547283>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=14.024979>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=10.626801>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.405037>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=21.145342>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.430626>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=12.013387>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.979961>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=7.4794083>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=11.377588>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=16.562672>,\n",
              "  <tf.Tensor: shape=(), dtype=float32, numpy=18.055851>,\n",
              "  ...]]"
            ]
          },
          "metadata": {},
          "execution_count": 299
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "od_loss_n = [np.mean([v.numpy() for v in x]) for x in od_loss]"
      ],
      "metadata": {
        "id": "GaUtzoTbUgYM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "od_loss_n "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n-LSdbHUfBDr",
        "outputId": "5eab4257-1fcd-4686-cd3e-37491bdcb218"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[6.038677, 13.5124235]"
            ]
          },
          "metadata": {},
          "execution_count": 298
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Fg5nixIUo0b",
        "outputId": "082298c4-e55c-4654-e70b-4d6e10f8bfc3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'HasAns_exact_1D_ip_0': 0.00946073793755913,\n",
              "  'HasAns_f1_1D_ip_0': 4.642681225498406,\n",
              "  'HasAns_total_1D_ip_0': 10570,\n",
              "  'best_exact_1D_ip_0': 0.00946073793755913,\n",
              "  'best_exact_thresh_1D_ip_0': 0.0,\n",
              "  'best_f1_1D_ip_0': 4.642681225498406,\n",
              "  'best_f1_thresh_1D_ip_0': 0.0,\n",
              "  'exact_1D_ip_0': 0.00946073793755913,\n",
              "  'f1_1D_ip_0': 4.642681225498406,\n",
              "  'total_1D_ip_0': 10570},\n",
              " {'HasAns_exact_1D_ip_1': 0.0,\n",
              "  'HasAns_f1_1D_ip_1': 5.300843653464699,\n",
              "  'HasAns_total_1D_ip_1': 10570,\n",
              "  'best_exact_1D_ip_1': 0.0,\n",
              "  'best_exact_thresh_1D_ip_1': 0.0,\n",
              "  'best_f1_1D_ip_1': 5.300843653464699,\n",
              "  'best_f1_thresh_1D_ip_1': 0.0,\n",
              "  'exact_1D_ip_1': 0.0,\n",
              "  'f1_1D_ip_1': 5.300843653464699,\n",
              "  'total_1D_ip_1': 10570},\n",
              " {'HasAns_exact_1D_ip_2': 0.0,\n",
              "  'HasAns_f1_1D_ip_2': 5.537899448390327,\n",
              "  'HasAns_total_1D_ip_2': 10570,\n",
              "  'best_exact_1D_ip_2': 0.0,\n",
              "  'best_exact_thresh_1D_ip_2': 0.0,\n",
              "  'best_f1_1D_ip_2': 5.537899448390327,\n",
              "  'best_f1_thresh_1D_ip_2': 0.0,\n",
              "  'exact_1D_ip_2': 0.0,\n",
              "  'f1_1D_ip_2': 5.537899448390327,\n",
              "  'total_1D_ip_2': 10570},\n",
              " {'HasAns_exact_1D_ip_3': 0.01892147587511826,\n",
              "  'HasAns_f1_1D_ip_3': 5.711596156689041,\n",
              "  'HasAns_total_1D_ip_3': 10570,\n",
              "  'best_exact_1D_ip_3': 0.01892147587511826,\n",
              "  'best_exact_thresh_1D_ip_3': 0.0,\n",
              "  'best_f1_1D_ip_3': 5.711596156689041,\n",
              "  'best_f1_thresh_1D_ip_3': 0.0,\n",
              "  'exact_1D_ip_3': 0.01892147587511826,\n",
              "  'f1_1D_ip_3': 5.711596156689041,\n",
              "  'total_1D_ip_3': 10570},\n",
              " {'HasAns_exact_1D_ip_4': 0.06622516556291391,\n",
              "  'HasAns_f1_1D_ip_4': 6.519546752562073,\n",
              "  'HasAns_total_1D_ip_4': 10570,\n",
              "  'best_exact_1D_ip_4': 0.06622516556291391,\n",
              "  'best_exact_thresh_1D_ip_4': 0.0,\n",
              "  'best_f1_1D_ip_4': 6.519546752562073,\n",
              "  'best_f1_thresh_1D_ip_4': 0.0,\n",
              "  'exact_1D_ip_4': 0.06622516556291391,\n",
              "  'f1_1D_ip_4': 6.519546752562073,\n",
              "  'total_1D_ip_4': 10570},\n",
              " {'HasAns_exact_1D_ip_5': 0.8325449385052034,\n",
              "  'HasAns_f1_1D_ip_5': 7.63530433444409,\n",
              "  'HasAns_total_1D_ip_5': 10570,\n",
              "  'best_exact_1D_ip_5': 0.8325449385052034,\n",
              "  'best_exact_thresh_1D_ip_5': 0.0,\n",
              "  'best_f1_1D_ip_5': 7.63530433444409,\n",
              "  'best_f1_thresh_1D_ip_5': 0.0,\n",
              "  'exact_1D_ip_5': 0.8325449385052034,\n",
              "  'f1_1D_ip_5': 7.63530433444409,\n",
              "  'total_1D_ip_5': 10570},\n",
              " {'HasAns_exact_1D_ip_6': 3.4626300851466416,\n",
              "  'HasAns_f1_1D_ip_6': 9.391735162383963,\n",
              "  'HasAns_total_1D_ip_6': 10570,\n",
              "  'best_exact_1D_ip_6': 3.4626300851466416,\n",
              "  'best_exact_thresh_1D_ip_6': 0.0,\n",
              "  'best_f1_1D_ip_6': 9.391735162383963,\n",
              "  'best_f1_thresh_1D_ip_6': 0.0,\n",
              "  'exact_1D_ip_6': 3.4626300851466416,\n",
              "  'f1_1D_ip_6': 9.391735162383963,\n",
              "  'total_1D_ip_6': 10570},\n",
              " {'HasAns_exact_1D_ip_7': 7.332071901608326,\n",
              "  'HasAns_f1_1D_ip_7': 12.930337978598288,\n",
              "  'HasAns_total_1D_ip_7': 10570,\n",
              "  'best_exact_1D_ip_7': 7.332071901608326,\n",
              "  'best_exact_thresh_1D_ip_7': 0.0,\n",
              "  'best_f1_1D_ip_7': 12.930337978598288,\n",
              "  'best_f1_thresh_1D_ip_7': 0.0,\n",
              "  'exact_1D_ip_7': 7.332071901608326,\n",
              "  'f1_1D_ip_7': 12.930337978598288,\n",
              "  'total_1D_ip_7': 10570},\n",
              " {'HasAns_exact_1D_ip_8': 13.216650898770103,\n",
              "  'HasAns_f1_1D_ip_8': 19.607655486199075,\n",
              "  'HasAns_total_1D_ip_8': 10570,\n",
              "  'best_exact_1D_ip_8': 13.216650898770103,\n",
              "  'best_exact_thresh_1D_ip_8': 0.0,\n",
              "  'best_f1_1D_ip_8': 19.607655486199075,\n",
              "  'best_f1_thresh_1D_ip_8': 0.0,\n",
              "  'exact_1D_ip_8': 13.216650898770103,\n",
              "  'f1_1D_ip_8': 19.607655486199075,\n",
              "  'total_1D_ip_8': 10570},\n",
              " {'HasAns_exact_1D_ip_9': 30.264900662251655,\n",
              "  'HasAns_f1_1D_ip_9': 38.73617031975681,\n",
              "  'HasAns_total_1D_ip_9': 10570,\n",
              "  'best_exact_1D_ip_9': 30.264900662251655,\n",
              "  'best_exact_thresh_1D_ip_9': 0.0,\n",
              "  'best_f1_1D_ip_9': 38.73617031975681,\n",
              "  'best_f1_thresh_1D_ip_9': 0.0,\n",
              "  'exact_1D_ip_9': 30.264900662251655,\n",
              "  'f1_1D_ip_9': 38.73617031975681,\n",
              "  'total_1D_ip_9': 10570},\n",
              " {'HasAns_exact_1D_ip_10': 51.01229895931883,\n",
              "  'HasAns_f1_1D_ip_10': 60.654385480257396,\n",
              "  'HasAns_total_1D_ip_10': 10570,\n",
              "  'best_exact_1D_ip_10': 51.01229895931883,\n",
              "  'best_exact_thresh_1D_ip_10': 0.0,\n",
              "  'best_f1_1D_ip_10': 60.654385480257396,\n",
              "  'best_f1_thresh_1D_ip_10': 0.0,\n",
              "  'exact_1D_ip_10': 51.01229895931883,\n",
              "  'f1_1D_ip_10': 60.654385480257396,\n",
              "  'total_1D_ip_10': 10570},\n",
              " {'HasAns_exact_1D_ip_11': 62.10974456007568,\n",
              "  'HasAns_f1_1D_ip_11': 71.55136963616609,\n",
              "  'HasAns_total_1D_ip_11': 10570,\n",
              "  'best_exact_1D_ip_11': 62.10974456007568,\n",
              "  'best_exact_thresh_1D_ip_11': 0.0,\n",
              "  'best_f1_1D_ip_11': 71.55136963616609,\n",
              "  'best_f1_thresh_1D_ip_11': 0.0,\n",
              "  'exact_1D_ip_11': 62.10974456007568,\n",
              "  'f1_1D_ip_11': 71.55136963616609,\n",
              "  'total_1D_ip_11': 10570},\n",
              " {'HasAns_exact_1D_ip_12': 66.69820245979186,\n",
              "  'HasAns_f1_1D_ip_12': 75.82777834389077,\n",
              "  'HasAns_total_1D_ip_12': 10570,\n",
              "  'best_exact_1D_ip_12': 66.69820245979186,\n",
              "  'best_exact_thresh_1D_ip_12': 0.0,\n",
              "  'best_f1_1D_ip_12': 75.82777834389077,\n",
              "  'best_f1_thresh_1D_ip_12': 0.0,\n",
              "  'exact_1D_ip_12': 66.69820245979186,\n",
              "  'f1_1D_ip_12': 75.82777834389077,\n",
              "  'total_1D_ip_12': 10570},\n",
              " {'HasAns_exact_1D_ip_13': 68.7511825922422,\n",
              "  'HasAns_f1_1D_ip_13': 77.83153965053803,\n",
              "  'HasAns_total_1D_ip_13': 10570,\n",
              "  'best_exact_1D_ip_13': 68.7511825922422,\n",
              "  'best_exact_thresh_1D_ip_13': 0.0,\n",
              "  'best_f1_1D_ip_13': 77.83153965053803,\n",
              "  'best_f1_thresh_1D_ip_13': 0.0,\n",
              "  'exact_1D_ip_13': 68.7511825922422,\n",
              "  'f1_1D_ip_13': 77.83153965053803,\n",
              "  'total_1D_ip_13': 10570},\n",
              " {'HasAns_exact_1D_ip_14': 69.3755912961211,\n",
              "  'HasAns_f1_1D_ip_14': 78.55207045688205,\n",
              "  'HasAns_total_1D_ip_14': 10570,\n",
              "  'best_exact_1D_ip_14': 69.3755912961211,\n",
              "  'best_exact_thresh_1D_ip_14': 0.0,\n",
              "  'best_f1_1D_ip_14': 78.55207045688205,\n",
              "  'best_f1_thresh_1D_ip_14': 0.0,\n",
              "  'exact_1D_ip_14': 69.3755912961211,\n",
              "  'f1_1D_ip_14': 78.55207045688205,\n",
              "  'total_1D_ip_14': 10570},\n",
              " {'HasAns_exact_1D_ip_15': 68.56196783349101,\n",
              "  'HasAns_f1_1D_ip_15': 78.13591462727811,\n",
              "  'HasAns_total_1D_ip_15': 10570,\n",
              "  'best_exact_1D_ip_15': 68.56196783349101,\n",
              "  'best_exact_thresh_1D_ip_15': 0.0,\n",
              "  'best_f1_1D_ip_15': 78.13591462727811,\n",
              "  'best_f1_thresh_1D_ip_15': 0.0,\n",
              "  'exact_1D_ip_15': 68.56196783349101,\n",
              "  'f1_1D_ip_15': 78.13591462727811,\n",
              "  'total_1D_ip_15': 10570},\n",
              " {'HasAns_exact_1D_ip_16': 66.66035950804162,\n",
              "  'HasAns_f1_1D_ip_16': 76.78253914985495,\n",
              "  'HasAns_total_1D_ip_16': 10570,\n",
              "  'best_exact_1D_ip_16': 66.66035950804162,\n",
              "  'best_exact_thresh_1D_ip_16': 0.0,\n",
              "  'best_f1_1D_ip_16': 76.78253914985495,\n",
              "  'best_f1_thresh_1D_ip_16': 0.0,\n",
              "  'exact_1D_ip_16': 66.66035950804162,\n",
              "  'f1_1D_ip_16': 76.78253914985495,\n",
              "  'total_1D_ip_16': 10570},\n",
              " {'HasAns_exact_1D_ip_17': 63.519394512771996,\n",
              "  'HasAns_f1_1D_ip_17': 74.3279648279204,\n",
              "  'HasAns_total_1D_ip_17': 10570,\n",
              "  'best_exact_1D_ip_17': 63.519394512771996,\n",
              "  'best_exact_thresh_1D_ip_17': 0.0,\n",
              "  'best_f1_1D_ip_17': 74.3279648279204,\n",
              "  'best_f1_thresh_1D_ip_17': 0.0,\n",
              "  'exact_1D_ip_17': 63.519394512771996,\n",
              "  'f1_1D_ip_17': 74.3279648279204,\n",
              "  'total_1D_ip_17': 10570},\n",
              " {'HasAns_exact_1D_ip_18': 58.72280037842952,\n",
              "  'HasAns_f1_1D_ip_18': 70.39730421556327,\n",
              "  'HasAns_total_1D_ip_18': 10570,\n",
              "  'best_exact_1D_ip_18': 58.72280037842952,\n",
              "  'best_exact_thresh_1D_ip_18': 0.0,\n",
              "  'best_f1_1D_ip_18': 70.39730421556327,\n",
              "  'best_f1_thresh_1D_ip_18': 0.0,\n",
              "  'exact_1D_ip_18': 58.72280037842952,\n",
              "  'f1_1D_ip_18': 70.39730421556327,\n",
              "  'total_1D_ip_18': 10570},\n",
              " {'HasAns_exact_1D_ip_19': 50.13245033112583,\n",
              "  'HasAns_f1_1D_ip_19': 62.69569523247236,\n",
              "  'HasAns_total_1D_ip_19': 10570,\n",
              "  'best_exact_1D_ip_19': 50.13245033112583,\n",
              "  'best_exact_thresh_1D_ip_19': 0.0,\n",
              "  'best_f1_1D_ip_19': 62.69569523247236,\n",
              "  'best_f1_thresh_1D_ip_19': 0.0,\n",
              "  'exact_1D_ip_19': 50.13245033112583,\n",
              "  'f1_1D_ip_19': 62.69569523247236,\n",
              "  'total_1D_ip_19': 10570}]"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "y_f1 = [y[i]['HasAns_f1' + '_1D_ip_' + str(i)] for i in range(0,20)]\n",
        "loss = [100 - y[i]['HasAns_exact' + '_1D_ip_' + str(i)] for i in range(0,20)]\n",
        "\n",
        "plt.rcParams[\"figure.figsize\"] = (10,6)\n",
        "plt.plot(x, y_f1, label = 'F1-score')\n",
        "plt.plot(x, loss, label = 'Loss')\n",
        "plt.xlabel('x')\n",
        "plt.ylabel('Score')\n",
        "plt.title('Distilbert on SQuAD dataset, 1-D interpolation (0 - Initial parameters, 1 - SQuAD minimizer)')\n",
        "plt.grid()\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 404
        },
        "id": "SE9KntOei5VG",
        "outputId": "5285327b-e4d8-4e95-8e25-1f51949a7f31"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAGDCAYAAACFuAwbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xUVdrA8d+TXgkk9NC7gBC6SguirmIvKBbEwiK66vq6vopuUVddcdd9LbiKHQSXYi+grqtEUER6B+mS0CEQkkBIO+8f5waHkDKBzNxk5vl+PvNJ5tbn3PrMuefeK8YYlFJKKaWUe0LcDkAppZRSKthpQqaUUkop5TJNyJRSSimlXKYJmVJKKaWUyzQhU0oppZRymSZkSimllFIuC7qETEQmisifT2G8FiKSIyKhzvc0ERnt/H+LiHxf3bGqqhGRbSJynttx1HQi8oiIvOF2HN4QkcdEZOppjL9GRFKrMSTPad8hIs/7YtpVjONGEflPBf0HisjPXk5Lj2WqRqjKvns6+7mIfCEio05l3Eqm+4GIXFSVcQIqIXNOyEdFJFtEDonIfBEZKyLHy2mMGWuMecLLaR0/uRtjthtj4owxRb6K32Pefj0oisjlIrJcRA6LyH4R+VZEWnv07ywin4pIlrNsvxWRs05hPreIiBGR60p1TxWRYifhzRGRDBGZKSJ9qqN85cRiRKSdr6Z/qvMRkSEiMsdZ1tu8GH6SiOQ76yVbRFaLyNMiklDeOMaYvxljRnsZz2klRP7kLIsnPbsZY7oYY9J8MK8I4E/APzy6pYjIEhE54vxNOY3pe73cjTHvGmMu8Bj3hG3OGDPPGNPxVGMJBG4mmiISISLvO+cUc7o/EESkroi8JSK7nX1+g4iM8+gvIvK/IrLROR9uF5G/OdtsVeYjIrJFRNaW0S9NRPKc+R92tvdxIhJ5OmXzVJV993T2c2PMRcaYyacybiWeAZ6sdCgPAZWQOS41xsQDLYHxwEPAm+6G5D0RCfPz/NoB7wB/ABKA1sC/gCKnf1vgB2CV068p8DHwtYj0reLsRgGZwM1l9NtpjIkD4oGzgPXAPBEZWtUy1XK5wFvA/1ZhnL8723wD4Fbs8vtBRGJ9EF+V+Ht79qPLgfXGmB1wPEH7BJgK1AMmA59U9SQYLGrbdlEN8X4P3ATsroZwngPigDOwx+zLgE0e/V8ExmCPs/HARcB5wPQqzmcQ0BBoU86P47ud404T7PljBDBbRKSK8wkoTiIbYoxZCNQRkd5ej2yMCZgPsA04r1S3vkAx0NX5Pgl40vm/PvA5cAibKMzDJqlTnHGOAjnAg0ArwABhzrhpwGjn/1uwSctLQBY2mRjqEUMCNincBezAZs2hpcZ9DjgAfADkYROiHOBQOWVtCnzqxL0J+K1Hv8eAmdhEKxtYA/QuZzrXAMsrWKZTgNlldH8FmOP8nwpkVLQusAlyMXA1UAg09uh30vhO95eAxRXENhL4xVluf/Scp7Pef3TW7S5nWhFOv7nOusx1lvF12JPo58A+4KDzfzOPed0CbHGW51bgRo9+twHrnPG+AlqWN58qbMvnAdu8GG4Szvbs0S3eKfPd5YzzGDDV+b+VE+MoYDuwH/ij0+9CIB8ocOJfcQrb85NUvn9Uti1P9fj+HvakluUs3y5O9zFOnPlOrJ+V3g6BSOB5YKfzeR6I9NwGsSeWvU7Zbq1gub8F/Mnj+wXOshCPbtuBC0/xWFa63AYYC2zEbtP/KpmXs3y/r2DbTsVj/wLGAZux2/Ja4MpS2/n35cRUsq2McZbfLuABj/7l7nMeZfidU4atTrcXgHTgMLAEGFhqGbyHTXKzsT8KOwAPO+soHbjAY/gyt0ts4nLSMdXZHp511tMeYCIQXWp7eAi7vU2hnPNFFddrBpB6KtuExzRWA1eU06+9U86+pbo3B44Bg53vaTjnr/LWO3Ybfxf4EHipVL8Txne6tQCOAJdUcKx6GfjCWQ8/AI2x++FB7HGhh8fw2/h1332MCs5pZQxble3meFmAFU5sJR9Tsr6wP3TnO+t/hed6dKbxlFOmo0A7p/vrwKPerttArCE7gbFZagYwsIzef3D6NQAaAY/YUcxI7E56qbGXKf/uxaz6YQ9y9YFHgQ9FJNHpNwmbhLQDemAP3qNLjbvFieEm7IH3R2fedcuZ33Qn9qbYpOpvInKuR//LnGHqYk92L5UznaVAJxF5zrlcFleq//nYjbu0mcBAEYkqZ7ql3YxNrj7AJi83ejHOh0DPsmp6RKQzNikciV0GSUAzj0GKgP/Bro+zgaHAXQDGmEHOMN2dZTwDm4i/jU0cW2B3qpececVif3VeZOwvwnOA5U6/y7HbzVXY7WgeMK2C+ficMSYb+Jqyt/nyDAA6YpfTX0TkDGPMl8DfgBlO/N2dYSfh/fb8lEe38vaPyrZlT19gTzoNsdvuu06ZX3P+/7sT66VljPtH7EE1BeiOTSD+5NG/MfakngzcDvxLROqVE8eZgGe7rC7ASuMchR0rne7V5RKgD9ANuBb4TekBvNzmNmO3jQTgcWCqiDSpQhxDsOvgAuAhj6Yd5e5zHq7Abgudne+LsOsjEfg38F6pY8ql2GSoHrAM+4MnBLuO/gq86jHsJMrYLo0x6yj7mDoee6JOccZJBv7iMb3GTlwtsUlomeeLSpaVLywAnhKRW0Wkfal+Q7HJ90LPjsaYdGe8C/CCiMRg98V3nc+Iymp7jTHbgcVUfNy5FrvP1ccmiD9i9+P6wPvA/1UwrrfnNKjaduNZhpL9Jg64H7uPLxWRZGAWNslPBB4APhCRBh6jj8RuJ/HYigKw57rueCngEzLHTuxCLK0AW93a0hhTYGxbi1PdwfYCzzvTmYFdkReLSCNgGHCfMSbXGLMXW3swwjM+Y8wEY0yhMeZoZTMSkeZAf+AhY0yeMWY58AYnXgr83hgz29g2b1MoZ6MwxmzB/hpMxiZZ+522OCWJWX3sL87SdmF/fZa1XMtyM/aAi/O3rMuWpe0EBLsDlnYN8LkxZq4x5hjwZ2wNHADGmCXGmAXOMt2G3QEHlzcjY8wBY8wHxpgjTkLzVKnhi4GuIhJtjNlljFnjdB8LPG2MWWeMKcQmMCki0tKL8vlSedt8eR43xhw1xqzA/vorc3s5je25vP3Dm235OGPMW8aYbGedPwZ0r6i9XCk3An81xuw1xuzDJiMjPfoXOP0LjDGzsb+Qy2t7VRf767tEHLbWzlMW9uBcXcYbYw45J7452ESiyowx7xljdhpjip11sRGbnHrrcWfdr8L+iLnema43+9zTxpjMku3CGDPV2fcKjTH/xNZaeS7zecaYr5x96z1sMjTeGFOAPTm3ctpUebNdHudcVhsD/I8TTzZ23/Ucvhhbu3HMibc6zxen4x5sknQ3sFZENsmvjcfLO17jdG9QTr/SrsImTP/BJiLhwMVejFfZcecjZzvJAz4C8owx7zjnqRnYRLo8Xp3THF5tN+WNLCIDsMnXZcaYw9iKktnO/IuNMV9jk89hHqNNMsascbblAqdbNmWfv8oULAlZMraKubR/YC+R/MdpvDiujGG8taPUzvkL9hd/S+zGvEvsjQaHsAeqhh7DpldxXk2BkoOI5/ySPb57tlU4AkSV1w7COYhea4xpgP11MwhbmwD2ElZZv56bYH8dHqgsWBHpj21/VtKG4d/AmV40ek525nGojH5N8Vhuxphcz1hEpIOIfO40fD2MPdjWryDGGBF5VUR+cYafC9QVkVBn2tdhk69dIjJLRDo5o7YEXvBYt5nYJDK5rPmcDrF3R5bc+DCxksHL2+bLU3p7KV1TWuJUt+fy9g9vtmUARCRURMaLyGZnHW1zepW7Xktpyq+/XD1jKHHAOYCXqGg5HOTEZCsHqFNqmDqcmLQBx+96LFmPa0r3r4C366hCInKz2Jt4StZfV7xfhnDi+j2+DL3c507YNkTkARFZJ/YmlkPYWjvPcfZ4/H8U2G9+vbGqJNmPw7vt0lMDIAZY4jH8l5yYsOxzEocS1Xm+KJM324bzw+lvxphe2CsDM7E1i4mUf7zG6b7fy1BGATOd5CIP25TGmzsRKzvulF6fpb9XtE17fU4rY7rlbTcncX4kzgRGGWM2OJ1bAsNLthVnexnAicu6rONePGWfv8oU8AmZ0xgxGduo8gTOL+0/GGPaYKtD7/doRF7VXz7JpRoztsD+WkjH/tKob4yp63zqGGM8L2WUnldl894JJIqI5wmhBbbdxGkxxizCXirs6nT6LzC8jEGvBRY4NRW52IMbYE+cnHhgG4VNUpaLyG7gJ4/uFbkSWOokRKXtwraLKJlnDPbgVOIVbJuE9saYOtjLCxU1Nv0D9pd5P2f4kks/AuD82jofuwOux7YNALt+7/BYt3WNMdHGmPmVlK3KnINwnPMZW95wTu3medjLp6c921LfT2V7hvL3j6psyzdgG9Ofhz1xt3K6l0zXm/3Gs+ayJIZTsRJ7uavEGqBbqTJ2c7qfwKlZKVmP1XlJs1JOze3r2NqVJGMv362m4n2jtOYe/3suQ2/2uePrSEQGYtvnXgvUc2LJqmIsJSrbLktvG/uxJ+YuHsMnGHup6qRYodLzRbWo6rbh1N78DYjF/uj9FmgupW64cpKMs7BtnaDUMRt7ebZk2GbAucBNTnK9G3tFYpiIVPSjtjnQi+o57rhCRKKxN609b4z5wqNXOjCl1HE+1hgz3mOYso4/Z2CvOHglYBMyEakjIpdga2WmOtXrpYe5RETaOQfRLGwbiJLLXnuANlWYZUPgXhEJF5Hh2BUx2xizC1vt+08nphARaSsi5V4+c+bdrLxr9sa2B5gPPC0iUSLSDdvmpcqPJxCRASLyWxFp6HzvhD3YLHAGeRw4R0SeEpFEEYkXkXuwd/OVtLfYgP21crGIhGPbCEQ604vCHnDHYC+xlHzuAW4o/QtHrGQReRTbLumRckJ/H7jEiT8C2y7Ac3uOxzYUznHKdGep8Uuv33jsAfqQ80vzUY+YGol9NEgs9qCfw6/byUTgYRHp4gyb4Kz/8uZT8liC1LIK5WwfUdhf++KsX6/u1BORSBHphT2gHMReTjpde7DV+yEAp7g9Q/n7R1W25Xjs8j+APZn8rYxYK9pnpwF/EpEGzonlL+XMxxuzOfFyXBr2+HGvsx7udrp/e4rTPx0VLYdY7IljH4CI3MqvP7689WexNcpdsMeBknZqle1zpcVj23ztA8JE5C+cXMvoFS+2yxOOqcaYYmxi+pzHsS9ZRE5ql1eiovOF2GYekyoYN1J+bRsX4Wzrp3Q3ooj8WUT6iH2cRhTwe2wtzM9Ojc5E4F0ROUtsrXIXbA3XfOwPbLBtYK9y1mM77D5XYiT2mN6RX4/XHbDt564vI54YZzl/AizE7hu11VvYu6dLtxufClwqIr9xlmmU2Mc1NStjGp4GY9u9eiUQE7LPRCQbm9H+EdtI8NZyhm2P3UBzsI0LXzbGzHH6PY09eB8SkQe8mO9PzvT2Y9sfXWOMKbmEdjMQgb2j6SA2maioEe232F/Wu0WkvCrm67E1BDux1+IfNcb8t5xhK3IIm4CtEpEcbLX9R8DfAYwxG7FVs92xl4gOAU9g78z6rzNMFrbx7hvYmo1c7M4LthHvUeAdY8zukg92ww/D3skH0NSZfw62oe+Z2LtYynzgpbFtuH6Hvfy5C7tcMzwGeQBbo5KNPfCWbtz8GDDZWb/XYu/0icauvwXOcigRgm3guRNbHT8Y52RjjPkI+7yZ6WIv06zG3mZe5nycX5Eld/6UZZCzvGbz680F5T700/Ggs80fwN6FtAQ4p5yaxaoquaHjgIgsdf6v6vYMFe8f3m7L72Avke1w5r2gVP83gc7Osv64jPGfxLb7WIld/kup4nOCPHyGvRmmKYAxJh+7rd+M3Uduw94Jl3+K0z8dj3Hitn2cMWYt8E/s8W4Pdj/7oYrT/w576e4b4FmPfbSyfa60r7D72Qbses2j6s03PFW0XZZ1TH3IKccCZ9/9L+W3GYSKzxfNqXg5/ozdl5Ox5T7KibW1VWGwP7b2Y/eZ84GLjTE5Tv+7scfiqdhLe6uxy/cKJxEF274uH7sNTMa5OcYxyinb7lLH7ImceFXjJee4swd7/PwAe1dxMbXXCOBK+fWycY6IDHR+OJbcwLUPu53+LxXkUGKvzuWYUjdYVKTktmmlvOL8IliAPWnWmue71RQichP2MsnDbsfiLyJyC/ZutwFux1KdRGQM0NkYc5/bsfiDiLTCPvIlvFRbu6Dm1LqtALp5NOauMUTkcWzzj0HGGK/bM6nTIyIfAG8ae4OQd+NoQqaqSkTOxP5aeN7jV5lSZQrUhCzYaEJWezmX0DcZ+ygbVUPVqqclq5rBaY9X3iU3pZRSNYgxpqJndqkaQmvIlFJKKaVcFoiN+pVSSimlahVNyJRSSimlXFar25DVr1/ftGrVyqfzyM3NJTb2pFcpBo1gLn8wlx2Cu/xa9uAsOwR3+YO57OCf8i9ZsmS/sW/FOUmtTshatWrF4sWLfTqPtLQ0UlNTfTqPmiyYyx/MZYfgLr+WPdXtMFwTzOUP5rKDf8ovIr+U108vWSqllFJKuUwTMqWUUkopl2lCppRSSinlslrdhkwppZRSvlVQUEBGRgZ5eXluh+JTCQkJrFu3rlqmFRUVRbNmzQgPD/d6HE3IlFJKKVWujIwM4uPjadWqFSLidjg+k52dTXx8/GlPxxjDgQMHyMjIoHXr1l6Pp5cslVJKKVWuvLw8kpKSAjoZq04iQlJSUpVrFDUhU0oppVSFNBmrmlNZXpqQKaWUUqpGCw0NJSUl5fhn27ZtHDhwgCFDhhAXF8fdd9/tdoinTduQKaWUUqpGi46OZvny5Sd0y83N5YknnmD16tWsXr3ab7EUFhYSFlb96ZPPashE5C0R2Ssiqz26JYrI1yKy0flbz+kuIvKiiGwSkZUi0tNXcSmllFKq9ouNjWXAgAFERUVVONyaNWvo27cvKSkpdOvWjY0bNwLwzjvv0K1bN7p3787IkSMB2LZtG+eeey7dunVj6NChbN++HYBbbrmFsWPH0q9fPx588EE2b97MhRdeSK9evRg4cCDr168/7fL4soZsEvAS8I5Ht3HAN8aY8SIyzvn+EHAR0N759ANecf4qpZRSqoZ4/LM1rN15uFqn2blpHR69tEuFwxw9epSUlBQAWrduzUcffeT19CdOnMjvf/97brzxRvLz8ykqKmLNmjU8+eSTzJ8/n/r165OZmQnAPffcw6hRoxg1ahRvvfUW9957Lx9//DFg7zadP38+oaGhDB06lIkTJ9K+fXt++ukn7rrrLr799ttTXAKWzxIyY8xcEWlVqvPlQKrz/2QgDZuQXQ68Y4wxwAIRqSsiTYwxu3wVn1dy91P34ErYUtI4z/zaz3j879m90n7lfnGJnPAHBI43RhTqZa6ALaWHlVL/V9JPQn6dbkl/CXG+e/YLObGf5zhl9is1jbAIiKzjEZNSSqlAUNYlS2+dffbZPPXUU2RkZHDVVVfRvn17vv32W4YPH079+vUBSExMJDs7mx9//JEPP/wQgJEjR/Lggw8en87w4cMJDQ0lJyeH+fPnM3z48OP9jh07dhqls/zdhqyRR5K1G2jk/J8MpHsMl+F0OykhE5ExwBiARo0akZaW5rNg6+9bQMqap2GFz2ZR43UHWOl2FN4rlnDyIxLIj6hLQXhd8iPqOv8nHP+/pF9BeJyTzJUtJyfHp9tXTRfM5deyp7kdhmuCufzllT0hIYHs7GwA7k9t4ZN5l0z/VIbJy8sjPz//eP/PPvuM8ePHAzBhwgQuvfRSunTpwldffcWFF17ICy+8cNI4AEVFRRhjyM7OJjw8nIKCguPfCwoKCAkJITs7m8OHD5OQkMC8efMqjC8vL69K25JrjfqNMUZEqlxFZIx5DXgNoHfv3sanb2bPPZNl4fH0cKpJgVK1L1JO99Pp50fHa/KMx3dzQr9ly5bSo0ePE4ctc7yK+jndj/9f7HwvPvH7Cf1MBf3KGs9AwVFCcvcRlbufqNy9kLMXctfA3n1QXHhy+SUUYhvYT1wDiG3o8bchKzJ30b3jUIhrCDH1ITS47oFJS0vDp/tXDaZlT3U7DNcEc/nLK/u6deuq5YGpp6u8GKKiooiIiDje/4YbbuCGG2443n/Lli3H24rt3buXTZs2MWzYMK688krGjRtHUlISmZmZhIeH079/f2bNmsXIkSOZNGkSgwYNIj4+nvDwcKKjo4mPjyc+Pp42bdrw5ZdfMnz4cIwxrFy5ku7du58UV48ePbwun7/PMHtKLkWKSBNgr9N9B9DcY7hmTjd3xSaRVbcLtOrvdiSuydp6DFqe43YYp6e4GPIOQe4+J0nbCzn7nL97IXe//X//Jvu30D7Mz9YOPuZMRCAm8dekrVFX6HgRtDgn6BI1pZSqKVq1asXhw4fJz8/n448/5j//+Q+dO3c+YZiZM2cyZcoUwsPDady4MY888giJiYn88Y9/ZPDgwYSGhtKjRw8mTJjAhAkTuPXWW/nHP/5BgwYNePvtt8uc77vvvsudd97Jk08+SUFBASNGjDgpIasqf59JPgVGAeOdv594dL9bRKZjG/Nnud5+TAWOkBCbTMUkQoOOFQ9rDBzLhtx9LP3+P/Rsn3xyApezFxa9CQtehqi60P4Cm5y1Ow+i6vinTEopFURycnLK7L5t27ZKxx03bhzjxo07qXtJ4/0S2dnZtGzZsszG+ZMmTTrhe+vWrfnyyy8rnXdV+CwhE5Fp2Ab89UUkA3gUm4jNFJHbgV+Aa53BZwPDgE3AEeBWX8WlVIVEbFIVVYfDCWdA59SyhzuWA1vmwPrZsOFLWDUTQsKh9UDoOMwmaAnN/Bq6Ukqp2suXd1leX06voWUMa4Df+SoWpapdZByccan9FBdB+k/w82yboM1+wH4ad4NOF9vkrHE3vftTKaVUubTxi1KnKyTUtrNreQ6c/wTs3wg/z4Kfv4C08ZD2NNRpZhOzTsOg5QD7iA6llFLKoQmZUtVJBBp0sJ8B/2Pbnm340iZny6bCotfts9LanWcvbbY/H6Lruh21Ukopl2lCppQvxTWAniPtJ/8IbP0O1s+ySdqaDyEkzNaslbQ7q9fK7YiVUkq5QBMypfwlIsYmXR0vso/i2LHYtjv7+Qv4cpz9NOxiL2t2vAia9LB3iCqllAp4mpAp5YaQEGje137OewwObLaJ2c+zYd4/Ye4/IL4JdLgQet4MyT3djlgppVwTFxdX7qMvAoUmZErVBElt4Zy77edIJmz4yiZnK2fCkrftjQDn3A3tf6O1ZkopFYD0yK5UTROTCCnXw3VT4A/r4IIn4eBWmDYC/tUXFr8NBUfdjlIppVy1fPlyzjrrLLp168aVV17JwYMHAXjxxRfp3Lkz3bp1Y8SIEQB89913pKSkkJKSQo8ePbx6d6a/aQ2ZUjVZVAKccw/0GwtrPoYfJ8Dn98G3T0Lf30Kf0RBb3+0olVLB4otxsHtV9U6z8Zlw0fgqj3bzzTczYcIEBg8ezF/+8hcef/xxnn/+ecaPH8/WrVuJjIzk0KFDADz77LP861//on///uTk5BAVFVW9ZagGWkOmVG0QGg7dhsOY72DUZ5Dcyz7f7Lku8Nnv7bPPlFIqSGRlZXHo0CEGDx4M2NcgzZ07F4Bu3bpx4403MnXqVMLCbL1T//79uf/++3nxxRc5dOjQ8e41Sc2LSClVPhFoPch+9v0MP74Ey6fBkknQ4SJbm9byHH0rgFLKN06hJsvfZs2axdy5c/nss8946qmnWLVqFePGjePiiy9m9uzZ9O/fn6+++opOnTq5HeoJtIZMqdqqQUe4bAL8z2oY/JB9fdOkYfD6EFj9ARQVuh2hUkr5REJCAvXq1WPevHkATJkyhcGDB1NcXEx6ejpDhgzhmWeeISsri5ycHDZv3syZZ57JQw89RJ8+fVi/fr3LJTiZ1pApVdvFNYQhj0D/+2DFNPjxX/D+bZDQAs4aax+bERnvdpRKKXXKjhw5QrNmzY5/v//++5k8eTJjx47lyJEjtGnThrfffpuioiJuuukmsrKyMMZw7733UrduXf785z8zZ84cQkJC6NKlCxdddJGLpSmbJmRKBYqIGOhzO/S6FTZ8AfNfgq8egbRnoNcoe2NAQrLbUSqlVJUVFxeX2X3BggUndfv+++9P6jZhwoRqj6m66SVLpQJNSAh0uhhu+wJGfwvtzrVtzV7oBh+OgV0r3Y5QKaVUKVpDplQga9YLhk+Cg9tgwURY+g6snAGtB9sbANqdpzcAKKVUDaA1ZEoFg3qt7N1R96+F8x63j8l49xp4+SxYOgUKj7kdoVJKBTVNyJQKJtF1YcB98PsVcOWrEBIOn94Nz3W179DUxEwpVQZjjNsh1Cqnsrw0IVMqGIVFQPcRMHYe3PwJNOkG3/wVXhtS/U/hVkrValFRURw4cECTMi8ZYzhw4ECV3wagbciUCmYi0CbVfjZ8BZ/cbZOyIQ8jxSnuxqaUqhGaNWtGRkYG+/btczsUn8rLy6u2VypFRUWd8JgOb2hCppSyOvwG7loAs+6Hb/5KSp2O0H0aJLV1OzKllIvCw8Np3bq122H4XFpaGj169HBt/nrJUin1q9gke1fm1W8ScyQDXukPC1+Hcp4BpJRSqnpoQqaUOpEInHkNi/pMsO/FnP0ATL0SsjLcjkwppQKWJmRKqTLlRybBTR/AJc9B+iJ4+RxYMR20Ya9SSlU7TciUUuUTgd63wZ3fQ8Mz4KM7YOZIyN3vdmRKKRVQNCFTSlUusQ3cOts+VHbDV/aBsutnuR2VUkoFDE3IlFLeCQm1D5UdkwbxjWH6DfDxXZCX5XZkSilV62lCppSqmkZd7EvLBz4AK6bZOzG3znU7KqWUqtU0IVNKVV1YBAz9M9z+NYRFwuRL4YuHIP+I25EppVStpAmZUurUNesNd8yDvnfATxPh1UGQscTtqJRSqtbRhEwpdXoiYmDY3+07MQuOwpvnw7dPQWG+25EppVStoQmZUqp6tEmFu+ZDt+tg7t/hjaGwd53bUSmlVK2gCZlSqvpEJcCVr8B178LhnfDqYPjhRSgucjsypZSq0TQhU0pVvzMusfWAv7IAACAASURBVC8qb38+fP1nmHQJZG51OyqllKqxNCFTSvlGXAO4bipcMRH2rLaPx1j8tr56SSmlyqAJmVLKd0Qg5Xq4c769I/Pz++Dd4ZC92+3IlFKqRtGETCnle3Wbw8iP4aJ/wLbvYerVUFzsdlRKKVVjaEKmlPKPkBDoNwYuf8lewlz7sdsRKaVUjaEJmVLKv7pcCQ06wXfP6N2XSinl0IRMKeVfIaEw+CHYtx7WfOR2NEopVSNoQqaU8r/OV0DDzlpLppRSDk3IlFL+FxJia8n2b4DVH7odjVJKuU4TMqWUO864DBp11VoypZRCEzKllFtKaskObIRV77sdjVJKuUoTMqWUezpdAo3OtLVkRYVuR6OUUq7RhEwp5Z6QEEgdB5mbYdV7bkejlFKu0YRMKeWuThdD424w9+9aS6aUClqakCml3CUCqQ9D5hZYNdPtaJRSyhWakCml3NfxImjSHb7TWjKlVHDShEwp5b6SWrKDW2HldLejUUopv9OETClVM3S4EJr2cGrJCtyORiml/EoTMqVUzVBSS3boF1gxze1olFLKrzQhU0rVHO0vgOReMPcfUJjvdjRKKeU3mpAppWqO47Vk22HFv92ORiml/MaVhExE/kdE1ojIahGZJiJRItJaRH4SkU0iMkNEItyITSnlsnbnQXJvmPus1pIppYKG3xMyEUkG7gV6G2O6AqHACOAZ4DljTDvgIHC7v2NTStUAIjDkYchKh+VT3Y5GKaX8wq1LlmFAtIiEATHALuBcoOQNw5OBK1yKTSnltrZDoVlfmPtPKDzmdjRKKeVzfk/IjDE7gGeB7dhELAtYAhwyxpQ8ETIDSPZ3bEqpGqKkluxwBiyb4nY0Sinlc2KM8e8MReoBHwDXAYeA97A1Y485lysRkebAF84lzdLjjwHGADRq1KjX9Om+fYhkTk4OcXFxPp1HTRbM5Q/mskMNKL8x9Fj2MJHH9vJTv1cxIeF+m7XrZXdRMJcdgrv8wVx28E/5hwwZssQY07usfmE+nXPZzgO2GmP2AYjIh0B/oK6IhDm1ZM2AHWWNbIx5DXgNoHfv3iY1NdWnwaalpeHredRkwVz+YC471JDytxwP71zO4Lht0Pe3fpttjSi7S4K57BDc5Q/msoP75XejDdl24CwRiRERAYYCa4E5wDXOMKOAT1yITSlVk7QeDC3OgXn/hII8t6NRSimf8XsNmTHmJxF5H1gKFALLsDVes4DpIvKk0+1Nf8emlKphRCB1HLxzGSydDP3ucDsiVUvkFRRxIDefAznHnL/5ZOYe40BOPkfyi4iJDCU2IozYyDBiI0Lt38hQNmQWUX9HFjERocRFhhETGUZMeCghIeJ2kVSAc+OSJcaYR4FHS3XeAvR1IRylVE3WehC07A/z/g96joLwKLcjUi7IKygiMzefzNx89uccI9NJskqSrszcfPbnnph0lSUiLISYiFCO5BeRX1hc9swWfn9Sp5iIUGIiwoiLLPkbZpM6J6Hz7BYXGXZ82IZ1omjbII6EaP+1gVS1kysJmVJKea3k6f2TL4Elk+CssW5HpKqRMYbtmUdYkZHF3sN5HMjNJzMnnwO5tmarJPHKOVZY5vjhoUJibARJsZEkxUXQOimGROf/pNgIkuIiSYyNoH5cBImxEcRFhmFby0B+YTFH84vIyS/kyLFCcvOLmL9wCe06dSE3v5DcY0UcyS8k51iR09+zWyGZuflszzzCkWNFTr9Cisu5T65BfCRtG8TSrmEcbRvYT7uGcTRJiDoejwpumpAppWq+1gOh1UD4/v+g1ygIj3Y7InWKCouKWbvrMIu2HWTxtkwW/3KQfdm/PmsuLMQmWDaJiqR5vZhykiv7f52osFNOaCLCQogICyEh5tfaq0ObQ0nt0viUpmeM4VhhMTnHCjlyrIicY4XsPHSUTfty2Lw3h837cvh0+U4O5/2aXMZEhNKmQSztnCStbUObqLVMiiEyLPSU4lC1kyZkSqnaIfVhmDQMFr8NZ9/ldjTKSznHClm2/eDxBGx5+qHjlxOb1YtmQLv69GpZj54t6pFcN5o60aeeYLlNRIgKDyUqPBScpyd0blqH82h0fBhjDPtz8tnkJGib9+WwaW8Oi7Yd5OPlO48PFyLQIjHmhBq1tg3jaNcg7oQEUgUOTciUUrVDq/62Pdn3z0GvWyAixu2IVBl2Z+Wx+JdMFm87yKJtmazbdZhiYxOMM5rU4drezendqh69WybSOCH42gOKCA3iI2kQH8nZbZNO6Hckv5At+3JtorY3x6lZy2Xuhv3kF/3a3q1+XMTxBK3k0me7hnE01cuftZomZEqp2iP1EXj7Qlj8Fpxzt9vRBL3iYsOmfTks2vZrApZx8CgA0eGh9GhRl7vPbU+fVvVIaV6X+Cit2alITEQYXZMT6JqccEL3omJDeuaRE2rUNu/LZdbKXWQdLTg+XLN60aR2bEBqh4ac0y6JmAg9xdcmuraUUrVHy7OhTSr88Dz0vhUiYt2OKKjkFRSxakfW8QRsyS8HjycE9eMi6dOqHrf2b02fVvU4o0kdwkPdel1yYAkNEVrVj6VV/ViGnnHi5c8Dufls3pvD+t3ZzNu4jw+W7GDqgu1EhIbQr00igzs0ILVjQ9o2iNXasxpOEzKlVO2S+gi8dQEsehP63+t2NAEt60gBi7Zlssi5BLkqI+v4pbO2DWK5qGtjerdKpHfLerRMitETvp+JCPXjIqkfF0m/NkmMOqcVxwqLWLT1IGk/7yVtwz6enLWOJ2eto1m9aIZ0bEhqxwac3VZrz2oiXSNKqdqlRT9oey788AL0uV1ryXxgb3Yer363hakLfuFYYTHhocKZyQnc2r8VvVsl0qtlPRJjI9wOU5UhMiyUAe3rM6B9ff4EpGceIW3DPr77eS/vL8lgyoJftPashtKETClV+6Q+Am+eBwtfhwH3uR1NwNh7OI9/rzvGd/+dQ2Gx4YqUZK7t3YzuzevaOwdVrdM8MYaRZ7Vk5Fkty609a54YTWqHhiTlF9I3v1Brz1yiS10pVfs07wPtzoP5L0Kf0RAZ53ZEtdrew3m88t1m/v3TdgqKirmqZzPuHtKOVvW19jGQVFZ7drSgiJdXfk2/1omkOpc329TX2jN/0YRMKVU7pT4MbwyFha/BwPvdjqZW2nM4j1fSNvPvhdspKjZc1SOZPrEHuHZYd7dDU35QuvbsjY/TOBjVhDk/7+WJz9fyxOccrz3Ttme+p0tWKVU7NesN7c63tWR9fwuR8W5HVGvszspj4ne/JmJX90zmd0Pa0TIplrS0NLfDUy6IDAulS/1QUlM786dLOpfd9iwshH6tEznvjEZc1TNZH2NSzTQhU0rVXqkPwxvnwk+vwqAH3I6mxtudlccraZuYtiid4mLD1T2b8bsh7WiRpA/ZVSeqqO3Zo5+u4dmvfuaGs1pwW//WNKoTfA/49QVNyJRStVezXtD+NzB/AvQdA1F13I6oRtqVdZRX0jYzfWE6xcZwTS+biDVP1ERMVa5027OVGYd4be4WXp+7hbe+38oVKcmMGdSG9o20lvp0aEKmlKrdUsfB60Ng4asw6H/djqZG2XnIJmIzFtlEbHjvZtyVqomYOj3dmtXlpRt6sv3AEd78fgszFqfz3pIMzu3UkDGD2tCvdaLeCHAKNCFTStVuyT2hw0Uw/yWnliyh8nEC3M5DR3k5bRMzF2VgMFzTqzl3pbbVRExVqxZJMTx+eVfuO68DUxb8wuT52xjx2gK6N0tgzKC2XNi1MaEhmph5SxMypVTtlzoOXhts25INftDtaFyz49BRXp6ziZmL0wEY3tsmYs3qaSKmfKdebAT3Dm3PmEFteH9JBm/M28Lv/r2UlkkxjB7Qmmt6NSc6Qp9jVxlNyJRStV/TFOh4Mfzo1JJF13U7Ir/KOHiEl9M2856TiF3buzl3DWlHct1olyNTwSQqPJSbzmrJ9X1b8PXa3Uz8bgt//mQNz/13IyPPasnNZ7ckKS7S7TBrLE3IlFKBIXUcvDoQfppo/w8CGQeP8K85m3l/STqCcF2f5tyZqomYcldoiHBh1yb8pktjFm07yGtzN/PCNxt5de5mhvdqzuiBrWmZpA8dLk0TMqVUYGjSDTpdAj++DP3GBnQtWXrmEV5O28R7izMIEWFEnxbcmdqWppqIqRpEROjbOpG+rRPZtDeb1+duZcaidN796Rcu7NqYMYPaktI8cPfTqtKETCkVOFIfhvWfw4KXYcgjbkfjE6/P3cIzX64nRIQb+tlErEmCJmKqZmvXMJ5nrunGHy7owNvztzF1wS/MXrWbvq0TuWNQG4Z0bEhIkN8AoAmZUipwNO4KZ1wGC16Bs+6E6HpuR1Stvlm3h6dmr+OCzo14/PIumoipWqdhnSgeurATvxvSjukLt/PW91u5ffJi2jeM47eD2nB5SlMiw4LzBoAQtwNQSqlqlToOjh2GH//ldiTVatv+XO6bsZwuTevw4vU9NBlTtVpcZBijB7bhuweH8Px1KYSGCA++v5KBz8zhlbTNZB0tcDtEv9OETCkVWBp1gc6Xw4KJcCTT7WiqxZH8Qu6YsoTQEGHiTb2ICg/OGgQVeMJDQ7iiRzJf/H4g79zWlw6N4nnmy/X0H/8tT81ay57DeW6H6DeakCmlAs/gcZCfYx+DUcsZYxj3wSo27s1mwvU99OGuKiCJCIM6NGDq6H58fs8Ahp7RkLd+2MawF+axIv2Q2+H5hSZkSqnA06gzdLnCPig294Db0ZyWt37YxqcrdvKHCzoysH0Dt8NRyue6JifwwogefHXfIGIiQ7n+9QWk/bzX7bB8ThMypVRgGjwO8nPtOy5rqQVbDvC32ev4TZdG3JXa1u1wlPKrdg3j+ODOc2hdP5bRkxfzwZIMt0PyKU3IlFKBqWEnaH8+LJkERbWvgfCurKPc7bx+5tnh3fVlzSooNYyPYvqYs+jXJpE/vLeCl9M2YYxxOyyf0IRMKRW4+oyGnD322WS1yLHCIu56dylH84t4bWQv4qPC3Q5JKdfER4Xz9i19uTylKX//8mce/2wtRcWBl5RpQqaUClztzoO6LWDRm25HUiV//Wwty7Yf4tnh3WnXMN7tcJRyXURYCM9dm8JvB7Zm0vxt3DttGXkFRW6HVa00IVNKBa6QUOh9G2ybB3vXux2NV2YuTufdn7YzdnBbLjqzidvhKFVjhIQIf7y4M3+6+AxmrdrFqLcWBtTzyjQhU0oFth4jITQCFtf8WrJVGVn86ePV9G+XxAMXdHA7HKVqpNED2/DCiBSWbj/Ida/+yO6swHhWmSZkSqnAFlsfulwJK6bDsRy3oylXZm4+Y6cuoUFcJC+O6EFYqB6elSrP5SnJvH1LX9Izj3D1K/PZtDfb7ZBOm+7xSqnA12e0fZ3SqvfcjqRMhUXF3DNtKftyjvHKTT1Jiot0OySlarwB7esz446zOVZYzNWv/MiSX2r3mzk0IVNKBb5mfaDxmbDoDaiBt8w/+58N/LDpAE9e0ZVuzeq6HY5StUbX5AQ+uuscEmMjuOH1n/h67R63QzplmpAppQKfiK0l27Ma0he6Hc0Jvli1i4nfbeaGfi24tndzt8NRqtZpnhjD+2PPplOTOtwxZTH//mm72yGdEk3IlFLB4czhEFnH1pLVEJv2ZvPAeytIaV6XRy/t7HY4StVaSXGRTPttPwZ3aMAjH63iua831LoHyGpCppQKDhGxkHIDrP0Ycva5HQ3ZeQWMmbKE6IhQXrmpJ5FhoW6HpFStFhMRxms392Z4r2a88M1GHvloFYVFxW6H5TVNyJRSwaP37VCUD8umuBqGMYYH3lvBLweO8NINPWmSEO1qPEoFivDQEP5+TTfuHtKOaQvTGTvVvvGiNtCETCkVPBp0gNaDYPHbUOzeQfqV7zbz1Zo9PHxRJ85qk+RaHEoFIhHhgd905InLu/DN+j3c+MYCDubmux1WpTQhU0oFlz6jIWs7bPzaldnP3bCPZ7/6mUu7N+X2Aa1diUGpYDDy7Fa8fENPVu88zDUT55Nx8IjbIVVIEzKlVHDpOAzim7jSuD898wj3Tl9G+4bxPHP1mYiI32NQKphcdGYTpt7ej33Zx7j6lfms23XY7ZDKpQmZUiq4hIZDr1tg038hc4vfZptXUMSd7y6hqNjw6shexESE+W3eSgWzvq0TeW/sOQjCtRN/5MfNB9wOqUyakCmlgk/PUSAhti2ZHxhj+ONHq1m94zDPX5dCq/qxfpmvUsrq2DieD+86h8YJUYx6ayGzVu5yO6STaEKmlAo+dZrAGZfYuy0Ljvp8dlN/2s4HSzP4/dD2DD2jkc/np5Q6WdO60bw39my6N0/g7mlLmfTDVrdDOoEmZEqp4NRnNBw9CGs+9ulslvxykL9+toYhHRvw+6HtfTovpVTF6sZEMOX2flzQuRGPfbaW8V+srzEPkNWETCkVnFoNhPodfNq4f292Hne9u4QmCdE8f10PQkK0Eb9SbosKD+XlG3txY78WTPxuM394bwUFNeABstqqVCkVnETsg2K/fAh2LoOmPap18gVFxdz97jKyjhbw0V19SYgJr9bpK6VOXWiI8OQVXWlcJ4p/fr2B/Tn53NDC3ZoyrSFTSgWv7iMgPAYWvVntk/7b7HUs3JbJM1d344wmdap9+kqp0yMi3DO0Pc9cfSY/bNrP51sKXI1Ha8iUUsEruq596fjKmXDBExBdr1om+8nyHbz9wzZu7d+Ky1OSq2WaSinfuK5PC1olxZK1daWrcWgNmVIquPUZDYVHYfm0apncul2HeeiDlfRtlcgjw86olmkqpXyrX5skIkLdbeOpCZlSKrg16QbN+sLiN+E077bKOlLAHVOWkBAdzks39iA8VA+xSinv6NFCKaX6jIYDm2Drd6c8ieJiw30zlrEr6ygv39iLhvFR1RigUirQuZKQiUhdEXlfRNaLyDoROVtEEkXkaxHZ6PytnsYcSilVmc6XQ0zSaT0C44VvNjLn53385dIu9Gqphy+lVNW4VUP2AvClMaYT0B1YB4wDvjHGtAe+cb4rpZTvhUdBj5GwfjZk7ajy6Iu3ZfLCNxu5plczburXwgcBKqUCnd8TMhFJAAYBbwIYY/KNMYeAy4HJzmCTgSv8HZtSKoj1vhVMMSydXPmwpUz+8RcSosN54vKuiOjDX5VSVedGDVlrYB/wtogsE5E3RCQWaGSMKXnb525AX/imlPKfeq2g/QWwZBJSXOj1aJm5+Xy1ejdX9kgmOiLUd/EppQKa+PsdTiLSG1gA9DfG/CQiLwCHgXuMMXU9hjtojDmpIYaIjAHGADRq1KjX9OnTfRpvTk4OcXFxPp1HTRbM5Q/mskNwlj/xwGK6rXqCJW3uJbvFUK/G+WpbAdPW5/NE/2iax9f++6SCcb17CubyB3PZwT/lHzJkyBJjTO+y+rmRkDUGFhhjWjnfB2Lbi7UDUo0xu0SkCZBmjOlY0bR69+5tFi9e7NN409LSSE1N9ek8arJgLn8wlx2CtPzFRfBiDw4ST737fqh0cGMMv3l+LjERYXz8u/5+CND3gnK9ewjm8gdz2cE/5ReRchMyv/+cM8bsBtJFpCTZGgqsBT4FRjndRgGf+Ds2pVSQCwmF3rdR79Bq2Lu+0sGXbj/Ehj05jOjT3A/BKaUCmVv16/cA74rISiAF+BswHjhfRDYC5znflVLKv3qMpFjC7YNiKzFj0XZiIkK5pHtTPwSmlApkrrzL0hizHCirys67RhtKKeUrsUnsbdifxsunwdBHIbLsNiXZeQV8tmIXl3VvSlykvhZYKXV6an8LVKWUqmY7mw6D/GxYNbPcYT5fuYujBUWM6KuXK5VSp08TMqWUKuVwnQ7QuBssKv/9ltMXbqdjo3hSmtcts79SSlWFJmRKKVWaiH2/5Z7VkP7TSb3X7jzMiowsruvTXB8Eq5SqFpqQKaVUWc68BiITyny/5czF6USEhXBVz2QXAlNKBSJNyJRSqiwRsZByPaz5GHL2He+cV1DEh0szuLBLY+rGRLgYoFIqkGhCppRS5el9OxQXwLJ3jnf6cvVuDucV6rPHlFLVShMypZQqT4MO0HoQLH7bPsUfmL5oOy0SYzirTZLLwSmlAonXCZmIRHs8XV8ppYJDn9GQlQ4b/8PW/bks2JLJdX2aExKijfmVUtXHq4RMRC4FlgNfOt9TRORTXwamlFI1QsdhEN8EFr3BjEXphIYI1/Rq5nZUSqkA420N2WNAX+AQHH/SfmsfxaSUUjVHaDj0ugU2/ZcfFy9mSMeGNKoT5XZUSqkA421CVmCMySrVreynJSqlVKDpOYpiCeWiY19wvT6ZXynlA96+gG2NiNwAhIpIe+BeYL7vwlJKqRqkThOWRJ/D9SaN2NZlv9tSKaVOh7c1ZPcAXYBjwL+BLOA+XwWllFI1yc5DR3nu0CASyCFsvTafVUpVv0pryEQkFJhljBkC/NH3ISmlVM3y/pIM5hd3pqBeO8IXvWEfGKuUUtWo0hoyY0wRUCwiCX6IRymlapTiYsOMRekMaNeA8LPGwI7FsHOZ22EppQKMt5csc4BVIvKmiLxY8vFlYEopVRN8v2k/Ow4d5bo+zaH7CAiPgUVvuh2WUirAeNuo/0Pno5RSQWXGonTqxYRzQZdGEBYK3a6FFTPggicgup7b4SmlAoRXNWTGmMnANGCJ8/m3000ppQLWgZxj/Gftbq7q2YzIsFDbsfftUHgUlk9zNzilVEDx9kn9qcBG4F/Ay8AGERnkw7iUUsp1Hy3bQUGRsZcrSzTpBs37waI3oLjYveCUUgHF2zZk/wQuMMYMNsYMAn4DPOe7sJRSyl3GGKYt3E7PFnXp0Cj+xJ59RkPmZtj6nTvBKaUCjrcJWbgx5ueSL8aYDUC4b0JSSin3LfnlIJv35TKiT4uTe3a+HGKSbC2ZUkpVA28TssUi8oaIpDqf14HFvgxMKaXcNH1ROnGRYVzcrcnJPcMioefN8PNsyNrh/+CUUgHH24TsTmAt9pVJ9zr/3+mroJRSyk1HCgyfr9zJpd2bEhtZzs3ovW4FY2DJJL/GppQKTN4mZGHAC8aYq4wxVwEvAqG+C0sppdyzYFcheQXFjOhTwYvE67WE9hfA0slQmO+/4JRSAcnbhOwbINrjezTw3+oPRyml3Dc3o5AzmtShW7NKXlDSZzTk7IH1n/snMKVUwPI2IYsyxuSUfHH+j/FNSEop5Z7VO7LYdtjWjolIxQO3Gwp1W+qT+5Wq7Ra+Tnj+IVdD8DYhyxWRniVfRKQ3cNQ3ISmllHtmLEonLASuSEmufOCQUOh9G/zyPexd5/vglFLVb87TMPsBknfMdjUMbxOy+4D3RGSeiMwDpgN3+y4spZTyv6P5RXy8fAd9GoWSEOPlk316jITQSK0lU6o2mvM0fDceUm5iW6sRroZSYUImIn1EpLExZhHQCZgBFABfAlv9EJ9SSvnNF6t3kZ1XyODmVXjMYmwSdLkSVkyHowd9F5xSqnp5JGNcNgHE2zoq36hs7q8CJbcPnQ08gn190kHgNR/GpZRSfjd9YTqtkmLoWK+KB+azf2ffb/nxXfo6JaVqg9LJWIi7yRhUnpCFGmMynf+vA14zxnxgjPkz0M63oSmllP9s3pfDwm2ZXNenReWN+Utr0g0ueMo+KHb+C74JUClVPWpgMgZeJGQiUvJUxKHAtx79ynlaolJK1T4zF6UTFiJc3cuLxvxl6XcHdLkKvvkrbJ1bvcEppapHDU3GoPKEbBrwnYh8gr2rch6AiLQDsnwcm1JK+UV+YTEfLM1g6BkNaRgfdWoTEbEH+KT28P5tcHhn9QaplDo9NTgZg0oSMmPMU8AfgEnAAGOM8RjvHt+GppRS/vHNuj3sz8kv+0XiVREZB9dNgfwj8N4tUFRQLfEppU5TDU/GwIvHXhhjFhhjPjLG5Hp022CMWerb0JRSyj+mL0qnSUIUgzo0OP2JNegIl0+A9J/g67+c/vSUUqenFiRj4P1zyJRSKiDtOHSUuRv3Mbx3c0JDqtiYvzxdr4Z+d8KCl2H1h9UzTaVU1dWSZAw0IVNKBbmZi9IBGN6rWfVO+Py/QvN+8Ok9sO/n6p22UqpytSgZA03IlFJBrKjY8N7idAa0q0/zxGp+PW9YBAyfBOHRMGMkHMupdBSlVDWpZckYaEKmlApi8zbuY2dW3uk35i9PnaZwzVtwYCN8di8cvy9KKeUztTAZA03IlFJBbMaidBJjIzi/cyPfzaT1IBj6F1j9ASzUF5wo5VPHk7Eba1UyBpqQKaWC1L7sY3y9dg9X90wmIszHh8L+90HHi+GrRyB9oW/npVSwqsXJGGhCppQKUh8uzaCw2HBdn+a+n5kIXPEyJDSHmaMgZ5/v56lUMDkpGQt1O6Iq04RMKRV0jDHMWJROn1b1aNcw3j8zja5rHxp7NBM+uA2KCv0zX6UCXQAkY6AJmVIqCC3cmsmW/blc56vG/OVpfCZc8px91+Wcp/w7b6UCUdr4gEjGQBMypVQQmrEonfjIMIad2dj/M0+5AXrdAt//H6yf7f/5KxUo0sZD2tMBkYyBJmRKqSCTdbSAWat2cVlKU2IiwtwJ4sJnoEkKfDQWMre4E4NStVmAJWOgCZlSKsh8unwHxwqLub6vny9XegqPgmvfsY39Z9wMBUfdi0Wp2iYAkzHQhEwpFUSMMUxbmE6XpnXompzgbjD1WsLVb8Ce1TDrD/rQWKW8EaDJGGhCppQKIqt3HGbtrsOM8MejLrzR/nwY/CAsfxeWvuN2NErVbAGcjIEmZEqpIDJ90XaiwkO4LCXZ7VB+NfghaHsuzP5f2LnM7WiUqpkCPBkDTciUUkHiSH4hnyzfybAzm5AQHe52OL8KCYWr3oC4hjDzZjiS6XZEStUsQZCMgSZkSqkgMWvlLnKOFfruReKnIzYJrp0M2bvhozuguNjtiJSqGYIkGQNNyJRSQWLGonTaNIilT6t6bodSXsrjHAAAIABJREFUtuRecOF42PgfmPdPt6NRyl3HsuHLh4MmGQNw6SE8SinlP5v2ZrP4l4M8MqwTIuJ2OOXrfZt9+ficp6BZL9u2TKlgUlQIy6bAnL9B7l7ofTsM+0fAJ2PgYg2ZiISKyDIR+dz53lpEfhKRTSIyQ0Qi3IpNKRVYpi9MJyxEuKpnM7dDqZiIfbVSw87w/u1wKN3tiJTyn43/hYkD4PP7ILENjP4WLvm/oEjGwN1Llr8H1nl8fwZ4zhjTDjgI3O5KVEqpgHKssIgPl+3g/M6NqB8X6XY4lYuIsS8hLy6E90ZB4TG3I1LKt/asgSlXwrtXQ2EeXDsFbvvS1hIHEVcSMhFpBlwMvOF8F+Bc4H1nkMnAFW7EppQKLP9du5fM3HxGuPlk/qpKagtXvAw7lsBXj7gdjVK+kb0HPr3H1ortWAq/eRp+txA6X2Zri4OMGBeeDi0i7wNPA/HAA8AtwAKndgwRaQ58YYzpWsa4Y4AxAI0aNeo1ffp0n8aak5NDXFycT+dRkwVz+YO57BA45f/HoqPszjX8Y3A0IV4e5GtK2dtsnkSL9I9Y1+l/2NM41S/zrClld0swl99fZQ8pOkbz9I9psf1DxBSyI3kYv7S8lsLweJ/PuyL+KP+QIUOWGGN6l9nTGOPXD3AJ8LLzfyrwOVAf2OQxTHNgdWXT6tWrl/G1OXPm+HweNVkwlz+Yy25MYJR/+4Hc/2/vzuOrqu/8j7++N3tIyEIgBLIAIjsiqywuoLZipxUdaaUVtdaltbXTecy0Y1v7qx078+jy6IzTma6KVotWrNpWam07qDCCAoJLCEpVJISELUAWyL7c7++P7wlckUCiuffc5f18PO7jnnvOucnnk5Mb3nzvud9jy+542t6z5q1+PS9qeu/qtPaBy639bqG1B7ZH5FtGTe8+SeT+w957d7e1rz1i7Y8mWHvXYGtXLbf28M7wfs9+iMSxB7baXjKNH29ZLgCuMMbsBlbh3qr8MZBrjOn51GcxsNeH2kQkjjy+tRpj4JOzouRSSf2VlAxLfwXpg+Gx66DtqN8ViXwwlS/AvRfBH26DwUVw41/cuZJDzvK7sqgR8UBmrf2GtbbYWjsKWAY8b629FlgLLPV2uwF4KtK1iUj86OoO8tutNVw0bigjczP8LueDyy6ETz4I9bvhqS/qIuQSWw69Db9ZBg99Alrr4er74aZnoWye35VFnWiaGPYO4J+MMTuBIcD9PtcjIjHsz9sPcOBoG5+JpZP5e1M2Hz5yN+z4I2z8id/ViJxZ82H401fhZ3Oh6kW49Dtw+1aYuhQC0RQ9ooevE8Naa9cB67zlXcAcP+sRkfhgrWXF+l2MLhjEpRML/S5nYMz7ElRvhjV3wYgZMGqB3xWJvF9nG2z+hbvaREczzLoRFn4DBhX4XVnU00z9IhJ3tuyup7ymke9eOYVAIE4+Pm8MLPkp1L4JD18Nc2+DBV+BjFy/KxNxb6VvfxKe+1do2APjFrtR3aHj/a4sZmjcUETizn3rd5GXmcLSaJ+Zv7/SB8P1T8HEj8OG/4QfT4MX/xs6W/2uTBLZns2w4lJ48iZIy3G/o595TGGsnxTIRCSuVB5u5tkdB1k+t4yM1Di85EpOMVy9Aj6/HopnwZr/B/8zE179tbsOoEik1O2C314PD3wUju6FJT+Dz/8fjFnod2UxSYFMROLK/Rt2kRIIcN28Mr9LCa+ic2D5k3DD05Bd5GY8//l8d+K/Pokp4dRaD3+9E34yB95ZAwu/CV9+BaZfmzDXnQwHnUMmInGjvrmDJ16p4crpIxiWne53OZEx+gK4+Vn429Pw3N3w2HIonu0+1TbqfL+rk3hS+zfYtgpeeRBaG2D6crj4W5A93O/K4oICmYjEjYc3VdHWGeTmC8b4XUpkGQMTPwHjLofy38Da78GDfwdjL4VL7nKjaSIfRNMhd7J++aOw/3UwSTDuMlj0TRg+1e/q4ooCmYjEhfaubh7aWMVF44YyrtDfa+L5JikZZlwPUz8JL9/nph745QUwZSlcfCfkJ1hQlQ+msw3e/jOUr4Kdz0KwC4qmweLvw5SrIWuY3xXGJQUyEYkLT72+j8NN7dySaKNjp5KSAQv+wYWzF38Mm34Ob/4BZt4IF/2L/kGV97OWnIY3YfXv4I0/QHsjZI+AebfDtGUwbKLfFcY9BTIRiXnWWu5fX8mE4dksGDvE73KiR0YuXHoXzLkV/u8HsPUBeP03bpLZ+V9202hIYjvyLmx7DMpXMb2hClIGwaQr4JxrYPSFOkk/ghTIRCTmvfDOYd46eIwffXIaxsTJRLADaXARfOK/3GjH2n+DF34IW1bAhV+F2TdDcprfFUoktdTBG793b0nWvAwYGLOQHcOvYuJVX4O0LL8rTEgKZCIS81as38Ww7DSumDbC71KiW8FYd6Hy+f/gZlT/6zfd25mLvulGRDQaEr+6OmDnGndy/tt/he4OGDrRzaY/9ZMweAQH161josKYbxTIRCSm7dh/lPXvHOZrl40nNVlTK/bJyBluNvV318Kz34E/3AYv/Q9c8m2wCTJdSCKwFva+6kLY9iehtQ4GDYXZt8C0a2D4Oe4TuhIVFMhEJKatWF9JRkoS155X6ncpseesRTD6ItjxFDz3XXh0GdMHT4Qx90DZPL+rkw+qYc/x88I4shOS02H8x2Dap+Gsi92ncSXq6KiISMw6eLSN1eV7+cycUnIzU/0uJzYFAjD5KpjwcXhtJen/ezf8arG7OPQl34bCyX5XKH3RdhTefMqFsKoNbl3Z+e4C9JOWQHqOv/XJGSmQiUjMeuil3XQFLZ87f7TfpcS+pBSY9Tk2N47kwtTtsOHH8PMFUDYfSue5+5I5kJagc7xFE2uhscadkF+9BWq2wP5yCHbCkLGw6FtwzqcgL84vHxZnFMhEJCa1dHTxyOY9XDZpOGVDBvldTtwIJqXBBf/s5izb9HN3IviGe2D9j9ws7cOnunDWE9QGFfhdcvzrbIV9r7vg1RPCmg64bckZMGI6zPsiTLwCRs7UeWExSoFMRGLS41traGzt5JYLNToWFpn5bnb/i++E9mMuDFS9BFUb3Xxmm37m9isYd2IErWw+5Opcvg/FWncOWI038lX9MhyocKNfALll7vqlxXOgZDYUTnGjmxLzFMhEJOZ0By33b6hkemkuM8vy/S4n/qVlu5PBz7rYPe5qdyM2VS/Cno1uZvdXH3LbBhe7DwSUzoOyBTB0vEZsTqezFfa95oJXTwhrOui2pWTCiBkw/3Z3wfji2brKQhxTIBORmLPmzQPsqWvh65dP8LuUxJScBqXnuRtAsBtq33SjZ1UvQuULUPG425aR74WzeVA6313oPFFHdKyFhirvvK+XXQg7uN1dKxIgbzSMWeiCV8kcGDZZn4hMIDrSIhJzVqyvpCQ/g8smD/e7FAE3oezwqe523q0ueNTtcqNnVS+521t/cvumDHJvtZXOdyFt5CxIzfS3/nCwFtoaXeCq2XLi5PvmWrc9ZZCbD27BV06Mful8vISmQCYiMeW1PfVsrarn2x+fRFJAb4VFJWNgyFnuNn25W3d0vwtoPSFt3fcAC4EUd1J6yRwYPMKNqGXmQ0aeW87Ic9fkjJarCHR1uFDVdBCaauHYAXffdPCkWy10tZ14Xv5ZMPaSE6NfQydq9EveQ78NIhJTVqyvJDs9mU/NLvG7FOmPwUUw5e/dDaC1Aao3u3C2ZyNs/uWJE9ffx7h5tDLyvLCWH7KcFxLict8b6NIG9+38NWuhtR6aasmt3wbbat8brEJDV2vdqb9GRj5kFUJ2oXuLNmuYe1wwzoWwTJ3rKKenQCYiMaO6roU/b9/PLReOIStNf75iWkYujLvM3QCCQWhvdBe+bm1wwaelzgWlVu++pc5bfxgOv+32a2/s/XuYpFOHuEAyNB86EbiaDrprOwLnApR7z09Od6Eqq9CN9pXNh+zhJ8JW1jDIGu4uR5SsiYnlw9FfNBGJGQ+8WEnAGD47f5TfpchACwS80a68/j2vu9MLcPWnD3Gt9W4y1QMV0N1+IlAVjDsRurKG8frO/Zx7wWK3ra8jbCIDQIFMRGJCY2snv91SzSemjaAoJ8PvciRaJKVA1lB3GwANR9ZBwdkD8rVE+iPgdwEiIn3x6Mt7aO7o5uYLNBGsiMQfBTIRiXodXUEefHE3888awuQRukiyiMQfBTIRiXp/qtjHgaNt3HLBGL9LEREJCwUyEYlq1lrue6GSscOyuGjcwJwnJCISbRTIRCSqbXz3CG/uP8rN548moIlgRSROKZCJSFS7b/0uCrJSuXL6SL9LEREJGwUyEYlaO2uPsfatQ1w3dxTpKVFy6RwRkTBQIBORqHX/hkrSkgMsn1vqdykiImGlQCYiUelwUztPvrqXv59RzJCsNL/LEREJKwUyEYlKKzdW0dEV5KbzNRGsiMQ/BTIRiTptnd2s3FTFJROGMXZYlt/liIiEnQKZiESd3726l7rmDm7WRLAikiAUyEQkqgSDlhUbdjFl5GDmjsn3uxwRkYhQIBORqLL2rVp2HWrmlgvGYIwmghWRxKBAJiJR5b71uyjKSedjU4v8LkVEJGIUyEQkamzf28imXXXcuGAUKUn68yQiiUN/8UQkaty3fhdZacksm6OJYEUksSiQiUhU2NfQytPb9nPN7BIGp6f4XY6ISEQpkIlIVHjwpd1Ya7lxwSi/SxERiTgFMhHxXVN7F49u3sPlU4sozsv0uxwRkYhTIBMR3z22pZpj7V3coolgRSRBKZCJiK+6uoM8sKGS2aPyOLck1+9yRER8oUAmIr76yxsH2NvQqsskiUhCUyATEd9Ya7lvfSWjhmRy6cRCv8sREfGNApmI+GZrVT3l1Q3cdP5okgK6TJKIJC4FMhHxzX0v7CI3M4WlM0v8LkVExFcKZCLii8rDzazZcZDl55WRkZrkdzkiIr5SIBMRXzywoZKUQIDr55f5XYqIiO8iHsiMMSXGmLXGmDeNMW8YY77irc83xqwxxrzj3edFujYRiYz65g4ef6WaJeeOYFh2ut/liIj4zo8Rsi7gn621k4C5wJeMMZOArwPPWWvPBp7zHotIHHpkcxVtnUFNdSEi4ol4ILPW7rfWvuotHwN2ACOBJcBD3m4PAVdGujYRCb/2rm4e2ljFBWcXMH54tt/liIhEBWOt9e+bGzMKeAGYAuyx1uZ66w1Q3/P4pOfcCtwKUFhYOHPVqlVhrbGpqYmsrKywfo9olsj9J3LvEL7+19d0cv/2Dr46K40pBckD/vUHQiIf+0TuHRK7/0TuHSLT/6JFi16x1s461Tbf/hoaY7KAJ4F/tNYedRnMsdZaY8wpk6K19l7gXoBZs2bZhQsXhrXOdevWEe7vEc0Suf9E7h3C0/8zFftZ9Xw5E4sG86Wrzyf0dR9NEvnYJ3LvkNj9J3Lv4H//vgQyY0wKLow9Yq39nbf6oDGmyFq73xhTBNT6UZuIDLyu7iA/+MvfuG99JeeW5PKza2dEbRgTEfFDxAOZ93bk/cAOa+1/hmxaDdwAfN+7fyrStYnIwKs91sbtv3mNlyvruG5uGd/6+ETSkjXvmIhIKD9GyBYA1wEVxpjXvXXfxAWx3xpjbgKqgE/5UJuIDKAtu+v44iOvcqytk3uumcZV04v9LklEJCpFPJBZazcAvb1XcUkkaxGR8LDW8sCLu/neMzsozstg5U1zmDB8sN9liYhErej8iJOIxKzm9i7ueHIbT2/bz0cmFfIfn5rG4PQUv8sSEYlqCmQiMmB21jbxhYdfYdehJv5l8Xi+cOFZBAI6eV9E5EwUyERkQDxTsZ+vPV5OekoSK286jwVjC/wuSUQkZiiQiciHEjqlxfRSN6VFUU6G32WJiMQUBTIR+cBCp7S4fl4Z3/q7SaQm+3GJXBGR2KZAJiIfSOiUFv91zblcOX2k3yWJiMQsBTIR6RdNaSEiMvAUyESkzzSlhYhIeCiQiUifhE5pccfiCXz+wjGa0kJEZIAokInIGYVOafHwTecxX1NaiIgMKAUyEelVZ3eQH2pKCxGRsFMgE5FTamgPcu19m3l5t6a0EBEJNwUyEXmfLbvruOulNtqD7ZrSQkQkAhTIROS40CkthqTDY7ct0JQWIiIRoEAmIsB7p7T46KRCriw6pjAmIhIhOiFEJMEFg5b17xxiyU9f5JmK/dyxeAK/vG4mmSma0kJEJFI0QiaSoBpaOnjilRoe2byHysPNDM1O05QWIiI+USATSTDl1Q2s3FTFH8v30d4VZFZZHl+55GwunzqctOQkv8sTEUlICmQiCaC1o5s/lu9j5aYqKvY2kpmaxNKZxSyfW8bEIp0nJiLiNwUykTj27qEmHtm0hydeqeZoWxfjCrO4e8lkrpo+kmxdg1JEJGookInEma7uIM/uOMjKTVW8uPMIKUmGxVOKWH5eKXNG52OMTtYXEYk2CmQiceJAYxuPvryHVVv2cPBoOyNzM/jaZeP51KwShman+V2eiIichgKZSAyz1vLSu0dYubGKNTsOErSWC88eyr9fWcaiCcNICmg0TEQkFiiQicSgxpZOnni1hkc2VbHrcDN5mSncfP5oPnNeKWVDBvldnoiI9JMCmUgMqahpZOWm3awu30dbZ5AZpbncc800Lp9SRHqKpqwQEYlVCmQiUa6t001Z8fCmKsprGslISeKq6cUsn1vK5BE5fpcnIiIDQIFMJMpYa6mpb6VibyMvV9bx+9f20tjaydhhWfzrFZO5asZIBmvKChGRuKJAJuKzg0fb2FbTyLaaBrbVNFKxt5G65g4AUpMCfGRyIdfNLeM8TVkhIhK3FMhEIuhIUzsVexu9ANZIxd4GDh5tByApYDh7WBYfmVjI1OIczinOYfzwbF3OSEQkASiQiYRJY2sn2/eeCF7l1Y3sbWgFwBgYUzCIBWcVHA9fk4pyyEhV+BIRSUQKZCIDoLm9izf2HWVbTcPxEbDKw83Ht5fmZzK9NJcb5pcxdWQuU0YO1qWLRETkOAUykX5q7ejmbweOUrG3kfJqN/q1s7aJoHXbi3LSOac4h6UzizmnOIepI3PIzUz1t2gREYlqCmQiJ+nqDrK/sY0dR7qp3VLNnroWqutbqK5robq+lUPH2o/vW5CVyjnFuVw+pYhpJTlMGZnDsOx0H6sXEZFYpEAmCcday5HmjuMBq7quJ2y1UF3Xyr6GVrp6hru2bCMpYCjKSackL5NF44dSmp/J2GFZnFOcS1FOuj75KCIiH5oCmcSl5vau4wGruq6FPXUt1PQ8rm+hpaP7PfsXZKVSnJfJtJJcPjGtiJK8TOqq3+HjC+dRlJtOSlLAp05ERCQRKJBJTAkGLQ2tnRw61u5uTW3Hl/c3tlFd30pNXQtHvHm8egxKTaIkP5OS/EwWjC2gJD+DkrxMb10Gmanvfymsa9lF6ZDMSLUmIiIJTIFMokJLR9eJkHWsnUNN7ad8fLipnc5u+77np6cEKBzs3lb86OTh7wlcpfmZ5GWm6K1FERGJWgpkMmCstXQFLV3dlo7uIF3dQTq6gxxp6jht0Ko92kbzSW8hAgQMFGSlMTTb3cYXZh9fHpqdxrDs9OPLg1KTFLhERCRmKZDFAGtdwOnocrd2775n3cmP3XL3e/Y/eR8Xlixd3UG6gpbO7iBd3ZauYMj6bsuR+lb+o2IDnd1Bt48XuHqWO7uCdAZ7nvv+katTGZyefDxITRmZw9Dxw7yAlfaewJWXmUpSQCFLRETinwJZmHV1B2lo7aS+uYO65g7qWzqpb+lwt+YO6po7aWjpoK6lg2NtXSGBKvie5YGSHDCkJgdISeq5GZKTDCmBgLtPCpCcFCAl4NanJ8PQ7DSSAz3bDMmBAKnJ7v74c7zt7uudeJyaHGDIoNTjIasgK430FM1GLyIiEkqBrB86u4M09ASqZheq6ppPPK5r6aChpdMLXm7d0bauXr9eRkoS+YNSyc1MIX9QKkU56aR6ISY1OUBqUhJpKYHj69KOrw/d58Rymvec49uST3puUoBAP0ec1q1bx8KFsz/sj05EREROQ4HsNDa+e4S7N7Zy15a11DW7EazeZKYmkZeZSt6gFPIyUynNzyR/UOp71oWGr7zMVI0UiYiICKBAdlppKQEykw1nleS6YJWZSv6gFHK9cBUathSuRERE5INSIDuNGaV5fHV2OgsXTve7FBEREYljmn5cRERExGcKZCIiIiI+UyATERER8ZkCmYiIiIjPFMhEREREfKZAJiIiIuIzBTIRERERnymQiYiIiPhMgUxERETEZ1EVyIwxi40xbxljdhpjvu53PSIiIiKREDWBzBiTBPwUuByYBHzaGDPJ36pEREREwi9qAhkwB9hprd1lre0AVgFLfK5JREREJOyMtdbvGgAwxiwFFltrb/YeXwecZ629/aT9bgVuBSgsLJy5atWqsNbV1NREVlZWWL9HNEvk/hO5d0js/tV7YvYOid1/IvcOkel/0aJFr1hrZ51qW3JYv3MYWGvvBe4FMMYcWrRoUVWYv2UBcDjM3yOaJXL/idw7JHb/6j1xJXL/idw7RKb/st42RFMg2wuUhDwu9tb1ylo7NKwVAcaYrb2l2USQyP0ncu+Q2P2r98TsHRK7/0TuHfzvP5rOIdsCnG2MGW2MSQWWAat9rklEREQk7KJmhMxa22WMuR34K5AEPGCtfcPnskRERETCLmoCGYC19hngGb/rOMm9fhfgs0TuP5F7h8TuX70nrkTuP5F7B5/7j5pPWYqIiIgkqmg6h0xEREQkISmQAcaYfGPMGmPMO959Xi/7dRtjXvduq0PWjzbGbPYu+fSY96GEmNGX/o0x5xpjNhpj3jDGbDPGXBOy7UFjTGXIz+bcyHbQf2e6TJcxJs07lju9YzsqZNs3vPVvGWMui2TdA6EPvf+TMeZN7zg/Z4wpC9l2ytdALOlD/581xhwK6fPmkG03eK+Td4wxN0S28g+vD73fE9L328aYhpBtMX3sjTEPGGNqjTHbe9lujDH/7f1sthljZoRsi/Xjfqber/V6rjDGvGSMmRaybbe3/nVjzNbIVT1w+tD/QmNMY8jv97dDtkXuko7W2oS/AT8Evu4tfx34QS/7NfWy/rfAMm/5F8Btfvc00P0D44CzveURwH4g13v8ILDU7z760W8S8C4wBkgFyoFJJ+3zReAX3vIy4DFveZK3fxow2vs6SX73NMC9LwIyveXbenr3Hp/yNRArtz72/1ngJ6d4bj6wy7vP85bz/O5pIHs/af8v4z5cFS/H/kJgBrC9l+0fA/4MGGAusDkejnsfe5/f0xPu8oWbQ7btBgr87iHM/S8Enj7F+n69Zj7sTSNkzhLgIW/5IeDKvj7RGGOAi4EnPsjzo8QZ+7fWvm2tfcdb3gfUAmGfBy5M+nKZrtCfyRPAJd6xXgKssta2W2srgZ3e14sVZ+zdWrvWWtviPdyEmxMwXnyYS7RdBqyx1tZZa+uBNcDiMNUZDv3t/dPAoxGpLAKstS8AdafZZQnwa+tsAnKNMUXE/nE/Y+/W2pe83iD+XvN9Ofa9ieglHRXInEJr7X5v+QBQ2Mt+6caYrcaYTcaYntAyBGiw1nZ5j2uAkWGsNRz62j8Axpg5uP8tvBuy+t+9Ie97jDFpYapzoIwEqkMen+qYHd/HO7aNuGPdl+dGs/7WfxNu1KDHqV4DsaSv/V/t/T4/YYzpmbA6YY699zb1aOD5kNWxfuzPpLefT6wf9/46+TVvgf81xrxi3KUL49U8Y0y5MebPxpjJ3rqIHvuomvYinIwxzwLDT7HpztAH1lprjOnto6dl1tq9xpgxwPPGmArcP9RRb4D6x/sf40rgBmtt0Fv9DVyQS8V9bPgO4O6BqFv8Y4xZDswCLgpZ/b7XgLX23VN/hZj1R+BRa227MebzuJHSi32uKdKWAU9Ya7tD1iXCsU9oxphFuEB2fsjq873jPgxYY4z5mzfiFE9exf1+NxljPgb8ATg70kUkzAiZtfZSa+2UU9yeAg56QaMncNT28jX2eve7gHXAdOAIbmi7J9ye8ZJPfhiI/o0xg4E/AXd6Q/o9X3u/N8zfDvyK6H8Lry+X6Tq+j3dsc3DHut+X+IoyfarfGHMpLqxf4R1XoNfXQCw5Y//W2iMhPa8AZvb1uVGuP/Uv46S3K+Pg2J9Jbz+fWD/ufWKMOQf3+77EWnukZ33Ica8Ffk/0/33vN2vtUWttk7f8DJBijCkgwsc+YQLZGawGej45cwPw1Mk7GGPyet6K8w7UAuBN6878WwssPd3zo1xf+k/FvRh/ba194qRtPWHO4M4/O+UnWaJIXy7TFfozWQo87x3r1cAy4z6FORr3v6iXI1T3QDhj78aY6cAvcWGsNmT9KV8DEat8YPSl/6KQh1cAO7zlvwIf9X4OecBHvXWxok+XpzPGTMCdvL4xZF08HPszWQ1c733aci7Q6J3KEevH/YyMMaXA74DrrLVvh6wfZIzJ7lnG9R7tf9/7zRgz3Pv3q+eUnADuP+CRvaRjuD4tEEs33LlBzwHvAM8C+d76WcAKb3k+UIH7lEUFcFPI88fg/lHeCTwOpPndUxj6Xw50Aq+H3M71tj3v/Uy2Aw8DWX731IeePwa8jTsP7k5v3d24EAKQ7h3Lnd6xHRPy3Du9570FXO53L2Ho/VngYMhxXu2t7/U1EEu3PvT/PeANr8+1wISQ537O+53YCdzody8D3bv3+DvA9096Xswfe9yI337v71gN7q25LwBf8LYb4Kfez6YCmBVHx/1Mva8A6kNe81u99WO8Y17uvSbu9LuXMPV/e8hrfhMwP+S573vNhOummfpFREREfKa3LEVERER8pkAmIiIi4jMFMhERERGfKZCJiIiI+EyBTERERMRnCmQiIiIiPlMgExEREfGZApmICGCMme1dUDzdm6H8DWPMFL/rEpHEoIlhRUQ8xph/w12lIQOosdZ+z+eSRCRBKJCJiHg0MwTsAAAAoUlEQVS869VtAdpwl0/p9rkkEUkQestSROSEIUAWkI0bKRMRiQiNkImIeIwxq4FVwGigyFp7u88liUiCSPa7ABGRaGCMuR7otNb+xhiTBLxkjLnYWvu837WJSPzTCJmIiIiIz3QOmYiIiIjPFMhEREREfKZAJiIiIuIzBTIRERERnymQiYiIiPhMgUxERETEZwpkIiIiIj5TIBMRERHx2f8HpFolEbqv+v4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y"
      ],
      "metadata": {
        "id": "8nIdtVIJRr8j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "y_arr = [list(item.values()) for item in y]\n",
        "new_array = np.array(y_arr)\n",
        "print(new_array)\n",
        "\n",
        "np.savetxt(\"./drive/My Drive/ydata_full_0-squad.txt\", new_array, delimiter =\", \")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GRlQIBhMIQay",
        "outputId": "cdb39d0c-f47f-4fd0-8e66-d4dad6d67de0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[9.46073794e-03 4.64268123e+00 1.05700000e+04 9.46073794e-03\n",
            "  4.64268123e+00 1.05700000e+04 9.46073794e-03 0.00000000e+00\n",
            "  4.64268123e+00 0.00000000e+00]\n",
            " [0.00000000e+00 5.30084365e+00 1.05700000e+04 0.00000000e+00\n",
            "  5.30084365e+00 1.05700000e+04 0.00000000e+00 0.00000000e+00\n",
            "  5.30084365e+00 0.00000000e+00]\n",
            " [0.00000000e+00 5.53789945e+00 1.05700000e+04 0.00000000e+00\n",
            "  5.53789945e+00 1.05700000e+04 0.00000000e+00 0.00000000e+00\n",
            "  5.53789945e+00 0.00000000e+00]\n",
            " [1.89214759e-02 5.71159616e+00 1.05700000e+04 1.89214759e-02\n",
            "  5.71159616e+00 1.05700000e+04 1.89214759e-02 0.00000000e+00\n",
            "  5.71159616e+00 0.00000000e+00]\n",
            " [6.62251656e-02 6.51954675e+00 1.05700000e+04 6.62251656e-02\n",
            "  6.51954675e+00 1.05700000e+04 6.62251656e-02 0.00000000e+00\n",
            "  6.51954675e+00 0.00000000e+00]\n",
            " [8.32544939e-01 7.63530433e+00 1.05700000e+04 8.32544939e-01\n",
            "  7.63530433e+00 1.05700000e+04 8.32544939e-01 0.00000000e+00\n",
            "  7.63530433e+00 0.00000000e+00]\n",
            " [3.46263009e+00 9.39173516e+00 1.05700000e+04 3.46263009e+00\n",
            "  9.39173516e+00 1.05700000e+04 3.46263009e+00 0.00000000e+00\n",
            "  9.39173516e+00 0.00000000e+00]\n",
            " [7.33207190e+00 1.29303380e+01 1.05700000e+04 7.33207190e+00\n",
            "  1.29303380e+01 1.05700000e+04 7.33207190e+00 0.00000000e+00\n",
            "  1.29303380e+01 0.00000000e+00]\n",
            " [1.32166509e+01 1.96076555e+01 1.05700000e+04 1.32166509e+01\n",
            "  1.96076555e+01 1.05700000e+04 1.32166509e+01 0.00000000e+00\n",
            "  1.96076555e+01 0.00000000e+00]\n",
            " [3.02649007e+01 3.87361703e+01 1.05700000e+04 3.02649007e+01\n",
            "  3.87361703e+01 1.05700000e+04 3.02649007e+01 0.00000000e+00\n",
            "  3.87361703e+01 0.00000000e+00]\n",
            " [5.10122990e+01 6.06543855e+01 1.05700000e+04 5.10122990e+01\n",
            "  6.06543855e+01 1.05700000e+04 5.10122990e+01 0.00000000e+00\n",
            "  6.06543855e+01 0.00000000e+00]\n",
            " [6.21097446e+01 7.15513696e+01 1.05700000e+04 6.21097446e+01\n",
            "  7.15513696e+01 1.05700000e+04 6.21097446e+01 0.00000000e+00\n",
            "  7.15513696e+01 0.00000000e+00]\n",
            " [6.66982025e+01 7.58277783e+01 1.05700000e+04 6.66982025e+01\n",
            "  7.58277783e+01 1.05700000e+04 6.66982025e+01 0.00000000e+00\n",
            "  7.58277783e+01 0.00000000e+00]\n",
            " [6.87511826e+01 7.78315397e+01 1.05700000e+04 6.87511826e+01\n",
            "  7.78315397e+01 1.05700000e+04 6.87511826e+01 0.00000000e+00\n",
            "  7.78315397e+01 0.00000000e+00]\n",
            " [6.93755913e+01 7.85520705e+01 1.05700000e+04 6.93755913e+01\n",
            "  7.85520705e+01 1.05700000e+04 6.93755913e+01 0.00000000e+00\n",
            "  7.85520705e+01 0.00000000e+00]\n",
            " [6.85619678e+01 7.81359146e+01 1.05700000e+04 6.85619678e+01\n",
            "  7.81359146e+01 1.05700000e+04 6.85619678e+01 0.00000000e+00\n",
            "  7.81359146e+01 0.00000000e+00]\n",
            " [6.66603595e+01 7.67825391e+01 1.05700000e+04 6.66603595e+01\n",
            "  7.67825391e+01 1.05700000e+04 6.66603595e+01 0.00000000e+00\n",
            "  7.67825391e+01 0.00000000e+00]\n",
            " [6.35193945e+01 7.43279648e+01 1.05700000e+04 6.35193945e+01\n",
            "  7.43279648e+01 1.05700000e+04 6.35193945e+01 0.00000000e+00\n",
            "  7.43279648e+01 0.00000000e+00]\n",
            " [5.87228004e+01 7.03973042e+01 1.05700000e+04 5.87228004e+01\n",
            "  7.03973042e+01 1.05700000e+04 5.87228004e+01 0.00000000e+00\n",
            "  7.03973042e+01 0.00000000e+00]\n",
            " [5.01324503e+01 6.26956952e+01 1.05700000e+04 5.01324503e+01\n",
            "  6.26956952e+01 1.05700000e+04 5.01324503e+01 0.00000000e+00\n",
            "  6.26956952e+01 0.00000000e+00]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2D interpolation"
      ],
      "metadata": {
        "id": "THOFhCqf_tWZ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bYAjrX6JrRSP"
      },
      "outputs": [],
      "source": [
        "def run_2D_interpolation(args, model_path_0, model_path_1, model_path_2, tokenizer_path,\n",
        "                         n, alpha_range_x, alpha_range_y):\n",
        "\n",
        "  model_0 = AutoModelForQuestionAnswering.from_pretrained(model_path_0) \n",
        "  model_1 = AutoModelForQuestionAnswering.from_pretrained(model_path_1) \n",
        "  model_2 = AutoModelForQuestionAnswering.from_pretrained(model_path_2) \n",
        "  tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)\n",
        "\n",
        "  #caching d/e/f, will skip if already done\n",
        "  dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True)\n",
        "\n",
        "  sd_0 = model_0.state_dict()\n",
        "  sd_1 = model_1.state_dict()\n",
        "  sd_2 = model_2.state_dict()\n",
        "\n",
        "  delta_1 = state_dict_diff(sd_0, sd_1)\n",
        "  delta_2 = state_dict_diff(sd_0, sd_2)\n",
        "  z = []\n",
        "\n",
        "  x = [alpha_range_x[0] + (alpha_range_x[1] - alpha_range_x[0]) * i / (n - 1.0) for i in range(0, n)]\n",
        "  y = [alpha_range_y[0] + (alpha_range_y[1] - alpha_range_y[0]) * i / (n - 1.0) for i in range(0, n)]\n",
        "\n",
        "  sqlen_1 = sum([torch.sum(torch.square(t)) for t in delta_1.values()])\n",
        "  sqlen_2 = sum([torch.sum(torch.square(t)) for t in delta_2.values()])\n",
        "  delta_2_scaled = state_dict_mult_n(delta_2, np.sqrt(sqlen_1/sqlen_2))\n",
        "\n",
        "  for i in tqdm(range(0, n)):\n",
        "    z_results = []\n",
        "\n",
        "    for j in tqdm(range(0, n)):\n",
        "      logger.info(\"Calculating node: {} / {}\".format(i * n + j, n * n - 1))\n",
        "      global_step = \"1D_ip_\" + str(i) + \"_\" + str(j)\n",
        "\n",
        "      sd_n = state_dict_sum_coef(\n",
        "          state_dict_sum_coef(sd_0, delta_1, x[i]),\n",
        "          delta_2_scaled, y[j])\n",
        "\n",
        "      model_n = AutoModelForQuestionAnswering.from_pretrained(model_path_0, state_dict = sd_n)\n",
        "      model_n.to(args.device)\n",
        "\n",
        "      # Evaluate\n",
        "      result = evaluate_for_batch(args, dataset, examples, features, model_n, tokenizer, prefix=global_step)\n",
        "      result = dict((k + (\"_{}\".format(global_step) if global_step else \"\"), v) for k, v in result.items())\n",
        "\n",
        "      z_results.append(result)\n",
        "    z.append(z_results)\n",
        "  return x, y, z"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run 2D interpolation"
      ],
      "metadata": {
        "id": "7TdHMEbdCyPL"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PlXBcND410H7"
      },
      "outputs": [],
      "source": [
        "model_path_0 = \"./drive/My Drive/distilbert-init/checkpoint-init\"\n",
        "model_path_1 = \"./drive/My Drive/distilbert-squad-100/checkpoint-48000\"\n",
        "model_path_2 = \"./drive/My Drive/distilbert-aqa-100/checkpoint-16000\"\n",
        "tokenizer_path = \"./drive/My Drive/distilbert-squad-100/checkpoint-48000\"\n",
        "\n",
        "x_2, y_2, z = run_2D_interpolation(args,\n",
        "                               model_path_0, model_path_1, model_path_2, tokenizer_path,\n",
        "                               n=10, alpha_range_x = [-1, 2], alpha_range_y = [-1, 2])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "z_arr = [] \n",
        "for i in range(0,10):\n",
        "  z_arr.extend([list(item.values()) for item in z[i]])\n",
        "\n",
        "new_array = np.array(z_arr)\n",
        "print(new_array)\n",
        "\n",
        "np.savetxt(\"./drive/My Drive/zdata_10.txt\", new_array, delimiter =\", \")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oji7JUzLXtKP",
        "outputId": "0e94cd6b-6999-4dd7-f0dd-a67d63e5f0f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[1.18906064e-01 6.76959687e+00 8.41000000e+02 1.18906064e-01\n",
            "  6.76959687e+00 8.41000000e+02 1.18906064e-01 0.00000000e+00\n",
            "  6.76959687e+00 0.00000000e+00]\n",
            " [0.00000000e+00 7.07027313e+00 8.41000000e+02 0.00000000e+00\n",
            "  7.07027313e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  7.07027313e+00 0.00000000e+00]\n",
            " [0.00000000e+00 5.97298387e+00 8.41000000e+02 0.00000000e+00\n",
            "  5.97298387e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  5.97298387e+00 0.00000000e+00]\n",
            " [0.00000000e+00 2.69064111e+00 8.41000000e+02 0.00000000e+00\n",
            "  2.69064111e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  2.69064111e+00 0.00000000e+00]\n",
            " [0.00000000e+00 5.85688106e+00 8.41000000e+02 0.00000000e+00\n",
            "  5.85688106e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  5.85688106e+00 0.00000000e+00]\n",
            " [1.18906064e-01 5.49195207e+00 8.41000000e+02 1.18906064e-01\n",
            "  5.49195207e+00 8.41000000e+02 1.18906064e-01 0.00000000e+00\n",
            "  5.49195207e+00 0.00000000e+00]\n",
            " [2.14030916e+00 9.63888043e+00 8.41000000e+02 2.14030916e+00\n",
            "  9.63888043e+00 8.41000000e+02 2.14030916e+00 0.00000000e+00\n",
            "  9.63888043e+00 0.00000000e+00]\n",
            " [1.78359096e+00 8.95559753e+00 8.41000000e+02 1.78359096e+00\n",
            "  8.95559753e+00 8.41000000e+02 1.78359096e+00 0.00000000e+00\n",
            "  8.95559753e+00 0.00000000e+00]\n",
            " [8.32342449e-01 7.07710797e+00 8.41000000e+02 8.32342449e-01\n",
            "  7.07710797e+00 8.41000000e+02 8.32342449e-01 0.00000000e+00\n",
            "  7.07710797e+00 0.00000000e+00]\n",
            " [5.94530321e-01 6.55152249e+00 8.41000000e+02 5.94530321e-01\n",
            "  6.55152249e+00 8.41000000e+02 5.94530321e-01 0.00000000e+00\n",
            "  6.55152249e+00 0.00000000e+00]\n",
            " [1.18906064e-01 4.45862385e+00 8.41000000e+02 1.18906064e-01\n",
            "  4.45862385e+00 8.41000000e+02 1.18906064e-01 0.00000000e+00\n",
            "  4.45862385e+00 0.00000000e+00]\n",
            " [0.00000000e+00 6.09717591e+00 8.41000000e+02 0.00000000e+00\n",
            "  6.09717591e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  6.09717591e+00 0.00000000e+00]\n",
            " [0.00000000e+00 4.26455092e+00 8.41000000e+02 0.00000000e+00\n",
            "  4.26455092e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  4.26455092e+00 0.00000000e+00]\n",
            " [0.00000000e+00 2.44751406e+00 8.41000000e+02 0.00000000e+00\n",
            "  2.44751406e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  2.44751406e+00 0.00000000e+00]\n",
            " [0.00000000e+00 6.12240684e+00 8.41000000e+02 0.00000000e+00\n",
            "  6.12240684e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  6.12240684e+00 0.00000000e+00]\n",
            " [1.30796671e+00 7.59705272e+00 8.41000000e+02 1.30796671e+00\n",
            "  7.59705272e+00 8.41000000e+02 1.30796671e+00 0.00000000e+00\n",
            "  7.59705272e+00 0.00000000e+00]\n",
            " [9.15576694e+00 1.88049438e+01 8.41000000e+02 9.15576694e+00\n",
            "  1.88049438e+01 8.41000000e+02 9.15576694e+00 0.00000000e+00\n",
            "  1.88049438e+01 0.00000000e+00]\n",
            " [4.39952438e+00 1.40615565e+01 8.41000000e+02 4.39952438e+00\n",
            "  1.40615565e+01 8.41000000e+02 4.39952438e+00 0.00000000e+00\n",
            "  1.40615565e+01 0.00000000e+00]\n",
            " [5.94530321e-01 5.90577097e+00 8.41000000e+02 5.94530321e-01\n",
            "  5.90577097e+00 8.41000000e+02 5.94530321e-01 0.00000000e+00\n",
            "  5.90577097e+00 0.00000000e+00]\n",
            " [7.13436385e-01 7.47294827e+00 8.41000000e+02 7.13436385e-01\n",
            "  7.47294827e+00 8.41000000e+02 7.13436385e-01 0.00000000e+00\n",
            "  7.47294827e+00 0.00000000e+00]\n",
            " [0.00000000e+00 5.46272308e+00 8.41000000e+02 0.00000000e+00\n",
            "  5.46272308e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  5.46272308e+00 0.00000000e+00]\n",
            " [0.00000000e+00 4.99230779e+00 8.41000000e+02 0.00000000e+00\n",
            "  4.99230779e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  4.99230779e+00 0.00000000e+00]\n",
            " [0.00000000e+00 2.90606760e+00 8.41000000e+02 0.00000000e+00\n",
            "  2.90606760e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  2.90606760e+00 0.00000000e+00]\n",
            " [0.00000000e+00 6.13742097e+00 8.41000000e+02 0.00000000e+00\n",
            "  6.13742097e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  6.13742097e+00 0.00000000e+00]\n",
            " [1.07015458e+00 7.42052126e+00 8.41000000e+02 1.07015458e+00\n",
            "  7.42052126e+00 8.41000000e+02 1.07015458e+00 0.00000000e+00\n",
            "  7.42052126e+00 0.00000000e+00]\n",
            " [1.84304400e+01 3.22452662e+01 8.41000000e+02 1.84304400e+01\n",
            "  3.22452662e+01 8.41000000e+02 1.84304400e+01 0.00000000e+00\n",
            "  3.22452662e+01 0.00000000e+00]\n",
            " [1.98573127e+01 3.18004394e+01 8.41000000e+02 1.98573127e+01\n",
            "  3.18004394e+01 8.41000000e+02 1.98573127e+01 0.00000000e+00\n",
            "  3.18004394e+01 0.00000000e+00]\n",
            " [6.30202140e+00 1.58028361e+01 8.41000000e+02 6.30202140e+00\n",
            "  1.58028361e+01 8.41000000e+02 6.30202140e+00 0.00000000e+00\n",
            "  1.58028361e+01 0.00000000e+00]\n",
            " [2.25921522e+00 9.04958975e+00 8.41000000e+02 2.25921522e+00\n",
            "  9.04958975e+00 8.41000000e+02 2.25921522e+00 0.00000000e+00\n",
            "  9.04958975e+00 0.00000000e+00]\n",
            " [3.56718193e-01 6.71775567e+00 8.41000000e+02 3.56718193e-01\n",
            "  6.71775567e+00 8.41000000e+02 3.56718193e-01 0.00000000e+00\n",
            "  6.71775567e+00 0.00000000e+00]\n",
            " [0.00000000e+00 5.81738275e+00 8.41000000e+02 0.00000000e+00\n",
            "  5.81738275e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  5.81738275e+00 0.00000000e+00]\n",
            " [0.00000000e+00 3.42600111e+00 8.41000000e+02 0.00000000e+00\n",
            "  3.42600111e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  3.42600111e+00 0.00000000e+00]\n",
            " [0.00000000e+00 4.93087137e+00 8.41000000e+02 0.00000000e+00\n",
            "  4.93087137e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  4.93087137e+00 0.00000000e+00]\n",
            " [4.75624257e-01 7.93693198e+00 8.41000000e+02 4.75624257e-01\n",
            "  7.93693198e+00 8.41000000e+02 4.75624257e-01 0.00000000e+00\n",
            "  7.93693198e+00 0.00000000e+00]\n",
            " [1.17717004e+01 1.94066162e+01 8.41000000e+02 1.17717004e+01\n",
            "  1.94066162e+01 8.41000000e+02 1.17717004e+01 0.00000000e+00\n",
            "  1.94066162e+01 0.00000000e+00]\n",
            " [4.11414982e+01 5.30763888e+01 8.41000000e+02 4.11414982e+01\n",
            "  5.30763888e+01 8.41000000e+02 4.11414982e+01 0.00000000e+00\n",
            "  5.30763888e+01 0.00000000e+00]\n",
            " [2.71105826e+01 3.90839587e+01 8.41000000e+02 2.71105826e+01\n",
            "  3.90839587e+01 8.41000000e+02 2.71105826e+01 0.00000000e+00\n",
            "  3.90839587e+01 0.00000000e+00]\n",
            " [1.01070155e+01 2.05175142e+01 8.41000000e+02 1.01070155e+01\n",
            "  2.05175142e+01 8.41000000e+02 1.01070155e+01 0.00000000e+00\n",
            "  2.05175142e+01 0.00000000e+00]\n",
            " [2.02140309e+00 9.71617167e+00 8.41000000e+02 2.02140309e+00\n",
            "  9.71617167e+00 8.41000000e+02 2.02140309e+00 0.00000000e+00\n",
            "  9.71617167e+00 0.00000000e+00]\n",
            " [1.07015458e+00 6.62750225e+00 8.41000000e+02 1.07015458e+00\n",
            "  6.62750225e+00 8.41000000e+02 1.07015458e+00 0.00000000e+00\n",
            "  6.62750225e+00 0.00000000e+00]\n",
            " [0.00000000e+00 1.63025646e+00 8.41000000e+02 0.00000000e+00\n",
            "  1.63025646e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  1.63025646e+00 0.00000000e+00]\n",
            " [0.00000000e+00 1.14817849e+00 8.41000000e+02 0.00000000e+00\n",
            "  1.14817849e+00 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  1.14817849e+00 0.00000000e+00]\n",
            " [4.75624257e-01 1.75880183e+00 8.41000000e+02 4.75624257e-01\n",
            "  1.75880183e+00 8.41000000e+02 4.75624257e-01 0.00000000e+00\n",
            "  1.75880183e+00 0.00000000e+00]\n",
            " [1.29607610e+01 1.93406505e+01 8.41000000e+02 1.29607610e+01\n",
            "  1.93406505e+01 8.41000000e+02 1.29607610e+01 0.00000000e+00\n",
            "  1.93406505e+01 0.00000000e+00]\n",
            " [5.46967895e+01 6.53579220e+01 8.41000000e+02 5.46967895e+01\n",
            "  6.53579220e+01 8.41000000e+02 5.46967895e+01 0.00000000e+00\n",
            "  6.53579220e+01 0.00000000e+00]\n",
            " [4.98216409e+01 6.21974733e+01 8.41000000e+02 4.98216409e+01\n",
            "  6.21974733e+01 8.41000000e+02 4.98216409e+01 0.00000000e+00\n",
            "  6.21974733e+01 0.00000000e+00]\n",
            " [3.03210464e+01 4.34327356e+01 8.41000000e+02 3.03210464e+01\n",
            "  4.34327356e+01 8.41000000e+02 3.03210464e+01 0.00000000e+00\n",
            "  4.34327356e+01 0.00000000e+00]\n",
            " [9.98810939e+00 2.15349687e+01 8.41000000e+02 9.98810939e+00\n",
            "  2.15349687e+01 8.41000000e+02 9.98810939e+00 0.00000000e+00\n",
            "  2.15349687e+01 0.00000000e+00]\n",
            " [2.37812128e+00 9.76751459e+00 8.41000000e+02 2.37812128e+00\n",
            "  9.76751459e+00 8.41000000e+02 2.37812128e+00 0.00000000e+00\n",
            "  9.76751459e+00 0.00000000e+00]\n",
            " [9.51248514e-01 6.14261324e+00 8.41000000e+02 9.51248514e-01\n",
            "  6.14261324e+00 8.41000000e+02 9.51248514e-01 0.00000000e+00\n",
            "  6.14261324e+00 0.00000000e+00]\n",
            " [0.00000000e+00 1.40826143e-01 8.41000000e+02 0.00000000e+00\n",
            "  1.40826143e-01 8.41000000e+02 0.00000000e+00 0.00000000e+00\n",
            "  1.40826143e-01 0.00000000e+00]\n",
            " [2.85374554e+00 8.52261187e+00 8.41000000e+02 2.85374554e+00\n",
            "  8.52261187e+00 8.41000000e+02 2.85374554e+00 0.00000000e+00\n",
            "  8.52261187e+00 0.00000000e+00]\n",
            " [1.86682521e+01 3.03237918e+01 8.41000000e+02 1.86682521e+01\n",
            "  3.03237918e+01 8.41000000e+02 1.86682521e+01 0.00000000e+00\n",
            "  3.03237918e+01 0.00000000e+00]\n",
            " [6.24256837e+01 7.25126417e+01 8.41000000e+02 6.24256837e+01\n",
            "  7.25126417e+01 8.41000000e+02 6.24256837e+01 0.00000000e+00\n",
            "  7.25126417e+01 0.00000000e+00]\n",
            " [6.76575505e+01 7.79531942e+01 8.41000000e+02 6.76575505e+01\n",
            "  7.79531942e+01 8.41000000e+02 6.76575505e+01 0.00000000e+00\n",
            "  7.79531942e+01 0.00000000e+00]\n",
            " [5.36266350e+01 6.55545337e+01 8.41000000e+02 5.36266350e+01\n",
            "  6.55545337e+01 8.41000000e+02 5.36266350e+01 0.00000000e+00\n",
            "  6.55545337e+01 0.00000000e+00]\n",
            " [2.91319857e+01 4.27042501e+01 8.41000000e+02 2.91319857e+01\n",
            "  4.27042501e+01 8.41000000e+02 2.91319857e+01 0.00000000e+00\n",
            "  4.27042501e+01 0.00000000e+00]\n",
            " [8.20451843e+00 1.79775182e+01 8.41000000e+02 8.20451843e+00\n",
            "  1.79775182e+01 8.41000000e+02 8.20451843e+00 0.00000000e+00\n",
            "  1.79775182e+01 0.00000000e+00]\n",
            " [1.90249703e+00 9.01384563e+00 8.41000000e+02 1.90249703e+00\n",
            "  9.01384563e+00 8.41000000e+02 1.90249703e+00 0.00000000e+00\n",
            "  9.01384563e+00 0.00000000e+00]\n",
            " [3.56718193e-01 5.14625952e+00 8.41000000e+02 3.56718193e-01\n",
            "  5.14625952e+00 8.41000000e+02 3.56718193e-01 0.00000000e+00\n",
            "  5.14625952e+00 0.00000000e+00]\n",
            " [4.63733650e+00 1.08770726e+01 8.41000000e+02 4.63733650e+00\n",
            "  1.08770726e+01 8.41000000e+02 4.63733650e+00 0.00000000e+00\n",
            "  1.08770726e+01 0.00000000e+00]\n",
            " [2.79429251e+01 4.01631666e+01 8.41000000e+02 2.79429251e+01\n",
            "  4.01631666e+01 8.41000000e+02 2.79429251e+01 0.00000000e+00\n",
            "  4.01631666e+01 0.00000000e+00]\n",
            " [6.15933413e+01 7.30567141e+01 8.41000000e+02 6.15933413e+01\n",
            "  7.30567141e+01 8.41000000e+02 6.15933413e+01 0.00000000e+00\n",
            "  7.30567141e+01 0.00000000e+00]\n",
            " [7.05112961e+01 8.04730384e+01 8.41000000e+02 7.05112961e+01\n",
            "  8.04730384e+01 8.41000000e+02 7.05112961e+01 0.00000000e+00\n",
            "  8.04730384e+01 0.00000000e+00]\n",
            " [6.63495838e+01 7.75972767e+01 8.41000000e+02 6.63495838e+01\n",
            "  7.75972767e+01 8.41000000e+02 6.63495838e+01 0.00000000e+00\n",
            "  7.75972767e+01 0.00000000e+00]\n",
            " [4.99405470e+01 6.28234855e+01 8.41000000e+02 4.99405470e+01\n",
            "  6.28234855e+01 8.41000000e+02 4.99405470e+01 0.00000000e+00\n",
            "  6.28234855e+01 0.00000000e+00]\n",
            " [2.00951249e+01 3.22666742e+01 8.41000000e+02 2.00951249e+01\n",
            "  3.22666742e+01 8.41000000e+02 2.00951249e+01 0.00000000e+00\n",
            "  3.22666742e+01 0.00000000e+00]\n",
            " [4.04280618e+00 1.13061403e+01 8.41000000e+02 4.04280618e+00\n",
            "  1.13061403e+01 8.41000000e+02 4.04280618e+00 0.00000000e+00\n",
            "  1.13061403e+01 0.00000000e+00]\n",
            " [8.32342449e-01 5.50904708e+00 8.41000000e+02 8.32342449e-01\n",
            "  5.50904708e+00 8.41000000e+02 8.32342449e-01 0.00000000e+00\n",
            "  5.50904708e+00 0.00000000e+00]\n",
            " [8.32342449e-01 6.79921327e+00 8.41000000e+02 8.32342449e-01\n",
            "  6.79921327e+00 8.41000000e+02 8.32342449e-01 0.00000000e+00\n",
            "  6.79921327e+00 0.00000000e+00]\n",
            " [1.47443520e+01 2.79449024e+01 8.41000000e+02 1.47443520e+01\n",
            "  2.79449024e+01 8.41000000e+02 1.47443520e+01 0.00000000e+00\n",
            "  2.79449024e+01 0.00000000e+00]\n",
            " [4.97027348e+01 6.34333049e+01 8.41000000e+02 4.97027348e+01\n",
            "  6.34333049e+01 8.41000000e+02 4.97027348e+01 0.00000000e+00\n",
            "  6.34333049e+01 0.00000000e+00]\n",
            " [6.34958383e+01 7.62909504e+01 8.41000000e+02 6.34958383e+01\n",
            "  7.62909504e+01 8.41000000e+02 6.34958383e+01 0.00000000e+00\n",
            "  7.62909504e+01 0.00000000e+00]\n",
            " [6.37336504e+01 7.50908240e+01 8.41000000e+02 6.37336504e+01\n",
            "  7.50908240e+01 8.41000000e+02 6.37336504e+01 0.00000000e+00\n",
            "  7.50908240e+01 0.00000000e+00]\n",
            " [5.29131986e+01 6.61454840e+01 8.41000000e+02 5.29131986e+01\n",
            "  6.61454840e+01 8.41000000e+02 5.29131986e+01 0.00000000e+00\n",
            "  6.61454840e+01 0.00000000e+00]\n",
            " [2.49702735e+01 3.62777189e+01 8.41000000e+02 2.49702735e+01\n",
            "  3.62777189e+01 8.41000000e+02 2.49702735e+01 0.00000000e+00\n",
            "  3.62777189e+01 0.00000000e+00]\n",
            " [5.11296076e+00 1.06171264e+01 8.41000000e+02 5.11296076e+00\n",
            "  1.06171264e+01 8.41000000e+02 5.11296076e+00 0.00000000e+00\n",
            "  1.06171264e+01 0.00000000e+00]\n",
            " [1.18906064e+00 5.98449715e+00 8.41000000e+02 1.18906064e+00\n",
            "  5.98449715e+00 8.41000000e+02 1.18906064e+00 0.00000000e+00\n",
            "  5.98449715e+00 0.00000000e+00]\n",
            " [4.75624257e-01 5.86906996e+00 8.41000000e+02 4.75624257e-01\n",
            "  5.86906996e+00 8.41000000e+02 4.75624257e-01 0.00000000e+00\n",
            "  5.86906996e+00 0.00000000e+00]\n",
            " [2.37812128e-01 6.98245422e+00 8.41000000e+02 2.37812128e-01\n",
            "  6.98245422e+00 8.41000000e+02 2.37812128e-01 0.00000000e+00\n",
            "  6.98245422e+00 0.00000000e+00]\n",
            " [2.11652794e+01 3.86937331e+01 8.41000000e+02 2.11652794e+01\n",
            "  3.86937331e+01 8.41000000e+02 2.11652794e+01 0.00000000e+00\n",
            "  3.86937331e+01 0.00000000e+00]\n",
            " [4.36385256e+01 5.97125018e+01 8.41000000e+02 4.36385256e+01\n",
            "  5.97125018e+01 8.41000000e+02 4.36385256e+01 0.00000000e+00\n",
            "  5.97125018e+01 0.00000000e+00]\n",
            " [4.39952438e+01 5.91057584e+01 8.41000000e+02 4.39952438e+01\n",
            "  5.91057584e+01 8.41000000e+02 4.39952438e+01 0.00000000e+00\n",
            "  5.91057584e+01 0.00000000e+00]\n",
            " [3.13912010e+01 4.49321557e+01 8.41000000e+02 3.13912010e+01\n",
            "  4.49321557e+01 8.41000000e+02 3.13912010e+01 0.00000000e+00\n",
            "  4.49321557e+01 0.00000000e+00]\n",
            " [1.26040428e+01 2.19211397e+01 8.41000000e+02 1.26040428e+01\n",
            "  2.19211397e+01 8.41000000e+02 1.26040428e+01 0.00000000e+00\n",
            "  2.19211397e+01 0.00000000e+00]\n",
            " [2.73483948e+00 7.38641075e+00 8.41000000e+02 2.73483948e+00\n",
            "  7.38641075e+00 8.41000000e+02 2.73483948e+00 0.00000000e+00\n",
            "  7.38641075e+00 0.00000000e+00]\n",
            " [3.56718193e-01 3.60254871e+00 8.41000000e+02 3.56718193e-01\n",
            "  3.60254871e+00 8.41000000e+02 3.56718193e-01 0.00000000e+00\n",
            "  3.60254871e+00 0.00000000e+00]\n",
            " [1.18906064e-01 4.25364751e+00 8.41000000e+02 1.18906064e-01\n",
            "  4.25364751e+00 8.41000000e+02 1.18906064e-01 0.00000000e+00\n",
            "  4.25364751e+00 0.00000000e+00]\n",
            " [3.56718193e-01 5.76856799e+00 8.41000000e+02 3.56718193e-01\n",
            "  5.76856799e+00 8.41000000e+02 3.56718193e-01 0.00000000e+00\n",
            "  5.76856799e+00 0.00000000e+00]\n",
            " [2.37812128e-01 6.83590635e+00 8.41000000e+02 2.37812128e-01\n",
            "  6.83590635e+00 8.41000000e+02 2.37812128e-01 0.00000000e+00\n",
            "  6.83590635e+00 0.00000000e+00]\n",
            " [7.37217598e+00 1.39889747e+01 8.41000000e+02 7.37217598e+00\n",
            "  1.39889747e+01 8.41000000e+02 7.37217598e+00 0.00000000e+00\n",
            "  1.39889747e+01 0.00000000e+00]\n",
            " [1.03448276e+01 2.08558109e+01 8.41000000e+02 1.03448276e+01\n",
            "  2.08558109e+01 8.41000000e+02 1.03448276e+01 0.00000000e+00\n",
            "  2.08558109e+01 0.00000000e+00]\n",
            " [6.89655172e+00 1.64246891e+01 8.41000000e+02 6.89655172e+00\n",
            "  1.64246891e+01 8.41000000e+02 6.89655172e+00 0.00000000e+00\n",
            "  1.64246891e+01 0.00000000e+00]\n",
            " [2.25921522e+00 7.90287717e+00 8.41000000e+02 2.25921522e+00\n",
            "  7.90287717e+00 8.41000000e+02 2.25921522e+00 0.00000000e+00\n",
            "  7.90287717e+00 0.00000000e+00]\n",
            " [4.75624257e-01 3.38531205e+00 8.41000000e+02 4.75624257e-01\n",
            "  3.38531205e+00 8.41000000e+02 4.75624257e-01 0.00000000e+00\n",
            "  3.38531205e+00 0.00000000e+00]\n",
            " [4.75624257e-01 3.35115387e+00 8.41000000e+02 4.75624257e-01\n",
            "  3.35115387e+00 8.41000000e+02 4.75624257e-01 0.00000000e+00\n",
            "  3.35115387e+00 0.00000000e+00]\n",
            " [4.75624257e-01 3.04221463e+00 8.41000000e+02 4.75624257e-01\n",
            "  3.04221463e+00 8.41000000e+02 4.75624257e-01 0.00000000e+00\n",
            "  3.04221463e+00 0.00000000e+00]\n",
            " [5.94530321e-01 5.60083179e+00 8.41000000e+02 5.94530321e-01\n",
            "  5.60083179e+00 8.41000000e+02 5.94530321e-01 0.00000000e+00\n",
            "  5.60083179e+00 0.00000000e+00]\n",
            " [3.56718193e-01 7.81670717e+00 8.41000000e+02 3.56718193e-01\n",
            "  7.81670717e+00 8.41000000e+02 3.56718193e-01 0.00000000e+00\n",
            "  7.81670717e+00 0.00000000e+00]\n",
            " [1.18906064e-01 6.08872835e+00 8.41000000e+02 1.18906064e-01\n",
            "  6.08872835e+00 8.41000000e+02 1.18906064e-01 0.00000000e+00\n",
            "  6.08872835e+00 0.00000000e+00]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def interpolate_list(arr, size):\n",
        "  res = np.zeros((2 * size - 1))\n",
        "  for i in range(0, size):\n",
        "    res[2 * i] = arr[i]\n",
        "  \n",
        "  for i in range(1, 2 * size - 1, 2):\n",
        "    res[i] = (res[i-1] + res[i + 1]) / 2\n",
        "\n",
        "  return res"
      ],
      "metadata": {
        "id": "o7j_LdFYUez4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def interpolate_array(arr, size):\n",
        "  res = np.zeros((2 * size - 1, 2 * size - 1))\n",
        "  for i in range(0, size):\n",
        "    for j in range(0, size):\n",
        "      res[2 * i, 2 * j] = arr[i,j]\n",
        "  \n",
        "  for i in range(1, 2 * size - 1, 2):\n",
        "    for j in range(0, size):\n",
        "      res[i, 2 * j] = (res[i-1, 2 * j] + res[i + 1, 2 * j]) / 2\n",
        "\n",
        "  for i in range(0, size):\n",
        "    for j in range(1, 2 * size - 1, 2):\n",
        "      res[2 * i, j] = (res[2 * i, j-1] + res[2 * i, j+1]) / 2\n",
        "\n",
        "  for i in range(1, 2 * size - 1, 2):\n",
        "    for j in range(1, 2 * size - 1, 2):\n",
        "      res[i, j] = (res[i-1, j-1] +\n",
        "                   res[i-1, j+1] + \n",
        "                   res[i+1, j-1] +\n",
        "                   res[i+1, j+1] ) / 4\n",
        "  \n",
        "  return res"
      ],
      "metadata": {
        "id": "fbNz2M43RAlH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.rcParams[\"figure.figsize\"] = (15,10)\n",
        "fig,ax=plt.subplots(1,1)\n",
        "\n",
        "z_f1 = [] \n",
        "for i in range(0,10):\n",
        "  z_f1.append(\n",
        "      [z[i][j]['HasAns_f1' + '_1D_ip_' + str(i) + '_'+ str(j)] for j in range(0,10)])\n",
        "  \n",
        "z_loss = [] \n",
        "for i in range(0,10):\n",
        "  z_loss.append(\n",
        "      [100 - z[i][j]['HasAns_exact' + '_1D_ip_' + str(i) + '_'+ str(j)] for j in range(0,10) ])\n",
        "  \n",
        "\n",
        "z_loss_np = interpolate_array(np.array(z_loss), 10)\n",
        "x_ip = interpolate_list(np.array(x_2), 10)\n",
        "y_ip = interpolate_list(np.array(y_2), 10)\n",
        "\n",
        "X, Y = np.meshgrid(x_ip, y_ip) \n",
        "\n",
        "cp = ax.contourf(X, Y, z_loss_np, levels = 15)\n",
        "fig.colorbar(cp) # Add a colorbar to a plot\n",
        "ax.set_title('Distilbert on SQuAD dataset, 2-D interpolation between initial parameters and SQuAD minimizer')\n",
        "ax.set_xlabel('x')\n",
        "ax.set_ylabel('y')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 621
        },
        "id": "LTgL7wGlF_at",
        "outputId": "b0dd2a23-8ba2-4b0a-ea29-ef4a9e57138a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1080x720 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAJcCAYAAAAivJGzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdefxeZ13n//cnLWlKWpKUAHZjsTJasFK1g+ioA6K/AXREFBGUTdRaFFdgFHWgyKaMiogOWATZFEUQB7AKzCgiIkjBQgpxaVnsRmnaJqShaZrm+v1xzknu7517Ocu1nvN6Ph73I/ne2zn3ft7n87muY845AQAAAEBJNqVeAQAAAADoiiADAAAAoDgEGQAAAADFIcgAAAAAKA5BBgAAAEBxCDIAAAAAikOQKYSZvcrM/meP293bzG41sxPqv99nZj9a//+pZvYB3+uKbszss2b27anXI3dm9ktm9gep16MNM7vYzN404PafNLOHelyl5n6Pfv7HqMvztu66ZvZXZvaUlvfFZzgjZnZfM3NmdmLqdUnFzL7FzP7V93UX3HbDNgYQG0EmA/WP4G1mtt/M9prZB83sIjM7+vo45y5yzr2g5X0d/UF1zv2Hc+4U59ydodZ/ZtlRg5GZPdrMLjezL5rZHjP7GzO738zlDzCzd5jZvvq5/Rsze0iP5Ty1/lH8gbnzH2pmR+ov8VvN7Boze4uZ/Wcfj2/Jujgz+4pQ9993OWb2bDO7on6eP2Nmz15z/deZ2aH6+vvr277EzLYtu41z7sXOuVYb4UODREz1c/HC2fOccw90zr0v0SotVEII6vK8zV530fvFOfdI59zr/a9lOcYa0Mzsm+vf2X1mdrOZ/cPs97aZnWVmf2RmN5nZATP7JzN7VI/lPLT+Lv2FufOboNX8dtxgZu8ys+/w8fgkyTn39865r/R93QW3jbaNASxCkMnHf3fOnSrpPpJ+TdIvSHpN2lVqL/aer3oj+w2Snilpm6T7Sfo9SXfWl58j6R8k7aovO0PSX0h6r5k9uOPiniLpZklPXnDZdc65UySdKukhkv5F0t+b2cO7PqbCmarnZ4ekR0h6hpk9fs1tXlq/5+8h6YdVPX//YGZbg65pC1Pek4t8lfS+tEp22xhmdjdJ75L0CkmnSTpT0vMl3V5ffpqkD0g6JOmBknZKepmkPzGz7+m4uFW/HZK0vf79eJCk90p6u5k9teMyRqek9zky4JzjlPgk6bOSvn3uvAdLOiLpq+u/XyfphfX/d6r6It6r6kvy71WF0jfWt7lN0q2S/oek+0pykk6sb/s+ST9a//+pqjb2f1fSPlUb4Q+fWYdtqsLU9ZKulfRCSSfM3fZlkm6S9DZJB1UFiVsl7V3yWM+Q9I56va+U9GMzl10s6S2qAsp+SZ+UdMGS+3mspMtXPKdvlHTpgvNfKelv6/8/VNI1q14LVcHyiKTvk3RY0pfNXHbc7evzf1fSZSvW7UmSPlc/b788u8z6df/H+rW9vr6vzfVl769fywP1c/wDqoLDuyTdKOmW+v9nzSzrqZI+XT+fn5H0QzOXPU3S7vp275Z0n2XL6fGe/h1Jr1hx+etUv59nzju1fszPWHKbiyW9qf7/fet1fIqk/5C0R9Iv15c9QtVGyB31+n+8x/v5hVr/+Vj3Xn7TzN9/Junz9f28X9ID6/MvrNfzUL2u75x/H0o6SdJvS7quPv22pJNm34OqAv0X6sf2wyue9/dJeomkf5L0RUn/R9JpM5c/RNIHVb3/Pi7pofX5L1L12T5Yr+fvqtr4e0V9+V3q98v/qv8+ub7uaavut+Xr8gFJv6HqffoZSY9s812qNd8nzXW1/P3yPh37rjxH0t+oem/skfRHqjZCl36Hz73XX6VqQ3W/pL9T/VmrL3+5pKvr1+Ojkr5l7n30Vklvqi//Ua34jqhv4yT9hKR/r5f3gnr9P1jfx1vmrv9dki6v7++Dkr5m5jt0w+9Ji9fyfareK/9Q3+4rtOI7aO55avO4Lqof115VO66svuyE+j2yp17WT2rmd29uORdoye9TffkLJF0hadPc+b9Q37dp7nd1/v1S/721fsyPV/X+mn3vHXf7+vxnSbphftl9XlvN/T6peo8+S9InVH0P/amkLSuu++z6ugdUfT7vJemv6uX+X0k75h+LpG+s3yvN6aCkz9bX2yTpFyVdpepz9BYd+35o7uNHVH2fv7/tbw0nTslXgNPyH8H6A/30+v+v07Eg8xJVP4x3qU/fMvOFvuG+5r8wdXyQOSzp5+r7+YH6C675cnm7pN+vv5DvqWrj58fnbvtT9RfYyfV5H1jzWN8v6X9L2iLpfFUb4N9WX3Zx/cX3KFU/TC+R9KEl9/Pl9XVfJulhkk6Zu/zzWrBBV1/3cL38DV/eS56//ynpn+r/75L0zJnLjrt9ff63qdoA2LrgsgfUX/DfqmoD9bfq9Wk2vr5e1YbCifVrt1vSz87c3kn6ipm/764qZN1VVRD4M0l/UV+2VdWP21fWf5+uYxvQj1a18X1uvaxfkfTBZcvp+H42Sf8s6aIV13md5oJMff4bJP3pkttcrOODzKvr996DVO1RPXf+ujO37/N+XvX5WPdeng0yT6tfnyaUXL7qudDGDfJflfShep3voWqj5QUz78HD9XXuouqz8yXVGxkLnsP3qQoLX10/D2+beU7PVLWB8ShVGx3fUf99j/nvjpn3+a76/9+kagPlwzOXfbzl/a57Xe6Q9GOqvhOerirM2brvUq35Pllw3fn3y9HHq2qD/Dvq1+8e9Wv/28u+Nxa81/fr2Gf+5Zr5npT0RFWf4xNVBdLP69gG5sX14/+e+rk7We2+I/6PpLupqijcLun/qfrO3CbpU5KeUl/3a1UF4G+on6On1I/lpEWPq8Vr+T5Vv1sPrNdvm5Z8By14nto8rndJ2i7p3qo+b4+oL7tI1Y6Gs1VVWf5Wy4PM3ep1fr2kR2rus6Lqs/b8Bbe7X32f91e7IPMkVYHsBEnv1MyOnUW3r8//8vr8c5c8R11e24fq+HDyT6p2wJxWP78Xrbjuh1SFlzPr98jH6vfLFlWh/nlrHstdVIX2l9R//0x9n2ep+hz8vqQ3z93HG1R9D5y87LeDE6f5U3ZlX2xwnaovnHl3qPpBuI9z7g5X9be6nsv4gqof5Ducc38q6V8lfaeZ3UvVj9XPOucOOOe+oCo0zLYLXeece4Vz7rBz7rZ1CzKzsyX9F0m/4Jw76Jy7XNIfaGPZ/QPOuUtd1W/7RlUbqMdxzn1a1Zfvmar27OypxxqcUl9lp6ofkXnND8ui53WRJ0v64/r/f6zlLQKzrlO1Mb99wWWPlfQu59z7nXO3qwpKR5oLnXMfdc59qH5OP6vqy/6/LluQc+4m59zbnHNfcs7tV7U3dPb6RyR9tZmd7Jy73jn3yfr8i1T9wOx2zh2W9GJJ55vZfVo8vnUuVrWR84c9brvsPb/M851ztznnPq5q7/DC98uA9/Oyz0eb9/JRzrnXOuf216/5xZIetGo80JwfkvSrzrkvOOduVFUJedLM5XfUl9/hnLtUVVBe1e/+RufcFc65A6ref4+rB+o+UVUV81Ln3BHn3HslXVY/b4v8o6T7m9ndVW2kv0bSmfVn8L+q2ojRqvtt+bp8zjn36vo74fWqvvvutfZZq7T6PlnHOXelc+69zrnb69fgt7Tic7nAX8585n9Z0jfW7yE5595Uf44PO+d+U9VG3uzr94/Oub+on7vbWn5HvNQ598X6836FpPc45z7tnNunaq/619bXu1DS7zvnPuycu9NVY4JuVxUoFmnzHnmdc+6T9ffKYS3/Dtqg5eP6NefcXufcf6gKK+fX5z9O1ef0aufczapC60LOuS9K+mYd2xFyYz2WsnlPrfrtkKog28ZTVO2UuVPVb8fjzewua25zXf3vqu/Atq/tIr/jnLuufo7eqWPP3yKvcM7d4Jy7VlXXx4edc//snDuoaufDquVIVVV+v6r3u1T95vyyc+6ame/Bx861kV1cfw+s3Z4AGgSZvJ2pqm1l3v9StTf9PWb2aTP7xQHLuHYuBH1O1R6b+6jao3J9PQHBXlU/LPecue7VHZd1hqSb6w3u2eWdOfP352f+/yVJW5b1y9Y/eo9zzt1DVVXqW3XsS3OPqg2eeaer+gG7ad3Kmtl/UbUX7k/qs/5Y0nlmturLX6oej1PV/jDvDM08b/XG5NF1MbP/VA/6/LyZfVFVwNi5Yh3vama/b2afq6//fknbzeyE+r5/QNUPyPVm9pdm9lX1Te8j6eUzr+3NqsLXmYuW05aZPUPVxvx31j9WzWxjzaDWV625i2Xv+WXm3y+nLLle3/fzss9Hm/eyJMnMTjCzXzOzq+rX6LP1RUtf1zln1Pc9vw6Nm+qNxsaq50Ha+Dg/p+p52anqOfr+5vmpn6Nv1uLPkeqNjctUbWx+q6rg8kFVAW82yKy63zavy9HX2Dn3pfq/qx7frNbfJ6uY2b3M7E/M7Nr6NXyT2r9+0sbP/K2q3uNn1Pf9LDPbXQ8836tqz/rORbetr9/mO+KGmf/ftuDv5vm7j6Rnzr02Z2vj+2tWm/fI/Pfbsu+gDVo+rmWf9w3fq9r4eTlOvQPnqc65s1RVJ89QVSmVVv92NJevVIfUh6lqQZSqKsoWSd+55qbN98eq78C2r+0ibb8vBy3HzH5c1Y7GH3TONTvq7qNqDFDzvtmtql11dqdE120KgCCTq3oGlTNV9YdvUO/ZfaZz7sslfbekn58ZXN61MnOmmdnM3/dWtVfoalV75nY657bXp7s55x44uyrzq7ZmWddJOs3MTp1b3rUd1/k4zrmPSPpzVT9KUtXD+/0Lrvo4Ve0lt6vq/b1rc0G9V3p2b9tTVG3cX25mn5f04ZnzV3mMpI/VP+Lzrle1odAs866q2koar1TVInF/59zdJP1SvQ7LPFPV3ttvqK//rc1dS5Jz7t3Oue9Q9SP8L6r2QErV6/vjM6/tdufcyc65D655bEuZ2dNU9UA/3Dl3TXO+q2YbO6U+XbTi9qeoGrPw933XYcb8e7HP+1la/vno8l7+QVWtfN+uaiP1vvX5zf22+dzMVsqadejr7Jn/31tVRWePqufojXPvia3OuV9bsZ5/p6qN7GslfaT++7+pGu/w/vo6q+63zesSw7rX4MX1dc6rP2dP1OrP5bzZz/wpqva4X2dm36JqLOPjVLU4bVfVvjh73/Pr1vU7YpWrJb1o7rW5q3PuzUuWve49ctxtVnwHzRvyuDZ8r6p6X7finPsXVe1/s78d37tgooLHqRqPdqWq3w5p5vdD0pfN/P9Jqrav3ln/dnxaVZBp89vxBVWV3yLV7+kXSHp0Xf1qXK1qfNvse2dLXfFp9O0swYQRZDJjZnczs+9SVQV4k3Nu14LrfJeZfUW9gbVP1V6NZq/HDar6Zdu6p6SfNrO7mNn3qxozcalz7npJ75H0m/U6bTKzc8xsVTvFDZLOMrPNiy50zl2tao/tS8xsi5l9jarBfZ2nybVq+swfM7N71n9/lapQ96H6Ks+X9E1m9iIzO83MTjWzn1I1O9Zz6+v8m6o9tN9Zl/x/RVVbh8xsi6ofrgtVld+b009J+sH5vbpWOdPMnqdqQO4vLVn1t0r6rnr9N6sa2zD7OTxVVU/5rfVjevrc7edf31NV7R3ba9VsO8+bWad7WTVF9VZVG4u36tj75FWSnmNmD6yvu61+/Zctp5mS+aGLHpSZ/ZCqjb3vcFXbX2tmdpKZfb2qWeVuUb+WtHk3SLpvszHS8/0sLf98dHkvn6rq+b9J1YbPixes66rP7Jsl/YqZ3cPMdqp6/w6ZWvqJVk1NfldV77+31u0vb5L0383sv9VVpC1WTR971or1/DtVFbhPOecOqR4nIOkzrmrB0qr7HfC6+Lbh/bLAqao+P/vM7ExVA6G7eNTMZ/4FqnamXF3f72FV4z1ONLPnqhr/sMq674guXi3pIjP7hvo7bGv9fdgE9PnXfN17ZIM130E+H9dbVH1OzzKzHap2qCxkZl9lZs9s1rmunjxBx347XqZ6Agoz+7L6MT5BVRvm81zVUnejqp0WT6yfh6epGnTfeIqq36DZ347vU/U+mN1x1azTvayqZj9P0nNmqhhFqZ/Lt0h6snPu3+YufpWkF1ndvlx/nz069jpifAgy+Xinme1Xtdfil1X1YP/wkuveX9Veo1tV9an/b+fc39aXvUTVRs9eM3tWi+V+uL6/ParGVzzWOde0Oj1Z0mZVAwhvUbURvrDNpPY3qmYG+ryZLSu/P0HVHunrVPXZPs85939brOe8vaqCyy4zu1XSX9f391JJcs79u6qWhwepauXZq2oD4jHN8uqe4p9QNbbhWlV72ZpKwveoCghvcM59vjlJeq2qwaiPqK93Rr38W1XtkT5P1Sw+71m00nVf80+qalO7XtXzes3MVZ6lag/+flUbGX86dxcXS3p9/fo+TlU7xMmqXr8P1c9DY5Okn1f1XN+sqt3n6fV6vF3Sr6uaUvSLqnqtH7lsOfUP1H5VEx4s8kJVlaWPWPs2sv9Rv+dvUjXI86OSvmlJJaurP6v/vcnMPlb/v+v7WVr9+Wj7Xn6DqlaXa+tlf2ju8tdIekD9XP/Fgtu/UFUL1ydUPf8fq8/r642q9kB/XtVe4p+Wju5oeLSqEH6jqu+iZ+vY78TLVfW032Jmv1Of90FV77+m+vIpVQPsm7/b3G+f18W3Re+XWc+X9HWqdhz9parqbxd/rGoj9WZVg9qfWJ//blWf2X9T9R45qPXtNeu+I1pzzl2maiKF31X13F+paoKFxobfkxav5byl30GeH9erVT2XH1f1+Vj1+uxXNbnBh83sgKrP4xWqqtuqP9/frOqz8SlV3+1vkPSTzrnXztzPj6l67DepGnj/QUmy6lhl95H0e7O/Hc65d6h6fp8wcx9763XYpWqc0ffPLaM0D1fVKvbWmd+BZkzUy1XN8vie+nv/Q6peB2CQZqYrYNTqvW8fUrWxWczxeXJhZk9UNdvQc1KvSyxWHc/hR51z35x6XVAuM3udqhmhfiX1uqA7q4478w+S3u6ce+666wOIi4oMJqEes/FISafbsZnN0JKrZlaaTIgBAOnoLGePknSnmX3ZuusDiCtZkDGzs83sb83sU2b2STP7mQXXMTP7HTO70sw+YWZfl2JdMQ7OuV3OuRe6atYgAADWctW0zs+v24sBZCRZa5mZnS7pdOfcx+qBhR+V9D3OuU/NXOdRqgZXP0pVL+XLnXP0VAIAAAATl6wi46oDY32s/v9+VXOKzx+D4dGqBls759yHVB0fI/YgUAAAAACZ6XxgsBDM7L6qjkPw4bmLztTGGVyuqc/bcNRdM7tQ1TS52nJX+/p7n7Nw9t9Jue3IsOfg5E2HPK0J+jrZipyBEwCAyfj4J+7Y46oDc2ft2x62xd18c5ztio9/4o53O+cesf6awyUPMvXA67dJ+tm5gye15py7RNIlkvSVX7PFvfId91lzi3HYdfDs9VcK7LwtHIg3hfM2M8wHAIDU7nnmdZ9LvQ5t3HzzEb33r+LkrXueed3OKAtS4iBj1UEI3ybpj5xzi+Z9v1Ybj9Z7ljwcBT53OQSUtvqsK+FnuF2HVk+8RtABAABjlyzImJmpOhDcbufcby252jskPcPM/kTVYP999ZGgi1RSQAmJ8BMeQQcAAIxdyorMf5H0JFVHZr+8Pu+XJN1bkpxzr5J0qaoZy66U9CUtP9J9dggtfvV9PglAi60KOoQcAABQgmRBxjn3AUm25jpO0k/GWaNhCC55IgB1RzUHAACUIPlg/1IRXMatzes71bBDNQcAAOSAINMCoQWLrHpfEHIWI+gAAABfCDILEFww1Lr3EEHneIQcAADQBUFG0wsuuw+cvvSyc7cWOylcUQg6x6OaAwAAuphckCG0+L3+LEKQPwSd41HNAQAAs0YfZKYUXIaEkNTLJwR1s+x9PcWAIxFyAACYotEFmduObJ5EeEkdWnwb+ngIQhUCzvFmQw6hBgCA8RhdkBmjsYWWEHw8R2MOQwScCqEGAIDxIMhkhtCSTtvnfkyBZ8oBh1ADAEDZCDIJEVrKtOx1I+CUa36MDcEGAID8EWQiIbSM3xSmtZ5KwKFaAwBA/ggygRFgII2/ijMfcMYUbAg1AADkiSATQAnh5ar9Oxeef86peyKvybSNNeCMNdgQagAAyAdBxqPcA8yy8NL1Om0RivobW5vaGIMNoQYAgLQIMh7kHGB8BpOUyyYUHbPo/VZauBlbsCHUAAAQH0FmgBwDTMrgEpLvxzW2YFR6uBlTsCHUAAAQB0GmI8LLOPR9zkoKQCWHm0Wzo5UYbgg1AACEQ5BpKacAQ3BJZ9FzT7iJo/SqDceqAQDAL4LMGrkEGMJLvgg3aYwp2BBqAADojiCzAOEFQ5U+vXWJ4abkYEOoAQCgO4LMjNQBhuAyfiVXb0oLN7PBhlADAMD4EGSUNsCkCi/X7du28Pwztu2LvCYouXpTSrhpQk1JgUYi1AAAsMqkg0yqAJNbeOl6nSEISu2VWr2Z/VzlFmpKrdJIhBoAAOZNLsgQXtKKuT5jDE2lhZv5z1tOwabUKo1EqAEAQJpQkEkRYAgvafl+HnINRiWFmxyrNSVXaSRCDQBgukYfZKZSgSG8hLfoOS4l3OQYbHINNSWGmUYTagg0AIApGHWQmUIVhgCT1vzzX0qwkfIKN81nNYdAU3LLWYNAAwCYgtEGmbGHGAJMnqjaDJNTlab06oxEoAEA5MnMfkbSj0kySa92zv22mV1cn3djfbVfcs5duup+Rhdkbjtyl9GGGMJLmUoJN7kFmxyqNGOozkhVoCHMAAByYGZfrSqwPFjSIUl/bWbvqi9+mXPuN9re1+iCTGwEGPRRQrjJJdjkUKWhOgMAgDfnSvqwc+5LkmRmfyfpe/vcEUFmgNAhJnSAObDv5OPO27rttqDLxHK5j7fJIdikrNKMqTojEWgAAMHsNLPLZv6+xDl3yczfV0h6kZndXdJtkh4l6TJJN0l6hpk9uf77mc65W1YtiCDTU8gQkyLAtLkslamGq9yrNimDTcoqzRiqMxKBBgCm5Da3acN0/YHtcc5dsOxC59xuM/t1Se+RdEDS5ZLulPRKSS+Q5Op/f1PS01YtiCDTUagAE6N9LMeQ0kYO651LmMq5atN8NmJXalJUacZSnZEINACA+Jxzr5H0GkkysxdLusY5d0NzuZm9WtK7ltz8KIJMByFCDAGmDPPPIcFmudnPydirNGOpzkhMCAAAiMfM7umc+4KZ3VvV+JiHmNnpzrnmB/wxqlrQViLItOQ7xBBgykawaWcKVZoxVWcAAIjkbfUYmTsk/aRzbq+ZvcLMzlfVWvZZST++7k4IMmv4DDCxZh8jwMRXQrBJGWqmEmhKDzNUZQAAMTjnvmXBeU/qej8EmRVKCzEEmHzkOCNcDtWalIGGMNMOYQYAUAqCzBK+QgwBBo3cqjYpg02KQBMzzEhlt5oRZgAAJSDIzCmpCkOAKRvBJn6godWsPcIMACB3BJkZpYQYXwFm097jX/4j2w97uW90N+VgkyLQUJ1ZjzADAMgZQaZWQitZyADT5rKpShXuchtnE2PigJiBJlaYkcquzhBmAAC5mvxWawlVmBgBBsvNP28pq1a5VG1CV2tiBRqmaW6HMAMAyNGm1CuQUu4h5sC+kwkxGdq098QNp5Sa90jq8VLX7dt29OTTVft3BjkQ7bzZg2mG1gSa0uw6dErqVQAAYIPJBpmcQ4zvAJN6Y3vscgk2OYUa32IEmthhpsRAQ5gBAORkclu4uQcYXwgv6cw+97mMr4ndgtZ8NkK1nElh2s5itppJZY+dAQAgtUlVZHINMVRgxmvq1ZoQ7WaNkFUaqjPLUZUBAORiMkEmxxBDgJmeHIJNilBTYqDZfeB0xs4sQZgBAORg9Fu+uQYYXwgvZUs9I1rsFrRQLWdSuJnOYk/TLJUxsxkzmQEAUht1RSa3EON7L/jQELP5lk1HT8hD6opNrGpNaRWamJUZqZzqDJUZAEBKo92dn1OI8b1R6CPAtDkPxzu040jU5aWcOGD2fRuqUlNShSbFRABS/tUZKjMAgFRGt/V6+50negsxPvYa+67AhAgxaG+2ihX7uUxZrZmt1ISo1oSu0PhEdeZ4VGYAACmwVbuEr1YyH3wFGEKMfynb88bYghYq0IwhzOQeaAgzAIDYRttaNkQuVRgfG6iEl3jmn+uYbWgpJw0I0YIWouWs9FYzKf/jztBmBgCIia3cOWMJMVRg0sulDS0m31WaEBUaqjNhUZkBAMTClm4tl/EwQzc+CTD5mlKoCRFofPI9s1nsMCOVMXYGAICQaC3zxFeIGYIAU47Z1ypVC1qM9rPmc+Gj5Ww2zPhqObtq/87iW80aObWcNVUZ2swAACERZIDEphBqfAYaye8YmtnKjI9QM1udSRVqpDyCzWybGaEGAOAbQQbISA6hRgoXbHxPDOB7UoCxhBopv2BDqAEA+EaQAeZs3nv8eYe2J1iPOtTEPginFKdaU0LbmVT2LGezcgo2tJ4BAHwgyNTO2LZv0IDirdtuGzROJsWxQMZsURjxcX8pA400zlCTc9uZFLZKI0072FClAQAMwdbzSIxxoL/vMOLD7DpNrUojhQ01odrOpHxDjZS2/WxWE2xSVWoINQCArggySC7HwNJGylCT8uCbjRihJtcqjeS/9UzKo1qTw0xohBoAQBsEGURVamhZJ2XrmTTe9rNQgUbKu0rTSF2tSV2lkY4/wCbBBgDQIMggmLGGllVSt55J6as1TagJEWgkQk0jxdTOqWc+kwg2AIBjCDIzhg74n7opBpdVcgg1Uh5TOudcpZHKmPVsXopqTQ5tZ/MINgAwXaMLMiedEP5o5blJMdCf0NJN6tazo+uRqFpTSpVGyn/Ws0VSVGtyqtLMItgAwHSMLsjAP0KLP7lUaRqxqzVUacKHGilutSbHKs0sgg0AjNcog8w5p+7ZsMGAbgguceQcaqTwwSZklcZnoJHGE2qksMEm1yrNLIINgCm67cjm444fFs6/RVrOSIMM2iO05CGX1rNZsao1Iao0odrOpDLH08yKUa3JvUozi2ADAOUiyEwYIQeSVvoAACAASURBVCY/uVVpGrFDTQljaaQyx9PMihlqcg80DYINAJSDIDOHmcuQixJCjRQm2IQeSyMxnmZeE2qo0mxEsAGAfI02yExlnEyKGcsQX46tZ43Q1ZoQVRqJULNM6EAjlVelmdUEGwINAKQ32iCD1WgrK9P865ZbsAkZakJVaaTwkwT4CjRS/EkCCDSL7Tp0CmEGABIjyAAFyznYxAg1U67SSPGPUUPb2UZUZwAgrVEHmam0lyGMLbe44847uMMSrEl7iyptOYSbJtSUWKWRCDUNqjSLUZ0BgDRGHWT66jvgf+u22zZs/KAsi4LLuuvkHmykvKo2JVZppPChxmegkcKHmpiBRioj1FCdAYD4CDITxPiYY9qEl7a3LyHUSPlUbUKFmpBVGilMqAlVpZHChpoYgUYqq0pDdQYA4iHIFIwZy7obGly63HcpwUZKX7UpsfVMItQ0YgcaKe9QQ3UGAOIYfZApYZzM7MYW/AsZXrosl2DTYrmFznomhQ01vgONFCbUxAo0UhlVGqozABAWW9ATM4W2slTBZZ0S29AaKdrRCDXHhKzSSMdCTcmBRsoz1FCdAYBwCDIYhVzDyzIlV2saMas2oVrPJELNrJIDjZR3lYbqDAD4R5BZou/MZYijtOCyztiCTahQE7JKI5Ubagg0G+VapaE6AwB+JR0tbmavNbMvmNkVSy5/qJntM7PL69Nz+ywn5NGvEc+WW9zR09iV/lg37z12CraMWzYdPYWwae+JR08hHNh3srfp2q/bt+3oyaer9u/0OsZw94HTNxxgM4ZdB8/eEGxy0AQaAMAwqSsyr5P0u5LesOI6f++c+644q1OOPhtvpY2PKXUj3rfSqzXN+67U1jOprANvhqjSlF6hkfJrO6M6AwDDJQ0yzrn3m9l9U64D8kJ4WW/Rc1RCuBlD65kUJ9TkOpZmTIFGyiPUMHYGAPpLXZFp4xvN7OOSrpP0LOfcJ+evYGYXSrpQkk75sq2RVw8+EGCGKS3cEGpWy32CgDEEGimfKg1hBgD6yf2Iih+TdB/n3IMkvULSXyy6knPuEufcBc65C7bsOGnhHTFOJl+EmDBmx9nMn3IyhvE0UrgxNT7H0kjyOpYmxBiaFHIYR8O4GQDoLusg45z7onPu1vr/l0q6i5lFO7pliOM1YKPcNqqnIteAEzvUxJgowFewyT3Q+JJiQoBG6jADAOgm6yBjZl9mZlb//8Gq1vemtGtVphwH+uew4YyNcqrixAg1UnnVmlwDjc8wI02zOkNVBgC6STpGxszeLOmhknaa2TWSnifpLpLknHuVpMdKerqZHZZ0m6THO+fY+h2BkkLMlpvvlCQdPO2ExGuS1rLXLMZYnBhjaqS442qGjqnxOTGAVAWaoVXosYydkapAk3rsDABgtdSzlj1hzeW/q2p6ZswIuec4hlJCTBNglv3dIODEnR46RaiR/AebHAONr6mbr9q/0+u4xN0HTk8WZqS4kwEw8B8A2it7i7gjBvynlcs4jHW23Hzn0tCy6vrzp6mK2Y4Wq/1MCteClmPLma9WszFMBCDFHztDixkAtFPC9MsYgVICTIz7m1IFJ2a1JlalRgrTgpZbhcZHq5nktzqTutVMSj9VMwDgmElVZGLw1a/uU+qB/rmHmNgVlClXcGJVa1JVanxUa3Kq0PicCIDqTMflUJUBgLUIMggq5xCTW4CYWsBJEWpKakHLLdD4QJgBAPhEaxmCyTXElBYOptCiNvteGctkAZKfCQNyaTnz2Wom+RmzmLLVLAYG/gPAapOryHT98eSgmP3kGGLGVuFYVsEpvaKTarKAmNWaPnKo0OR8EM3YqMoAQHpUZOBdbiGmxI15n9o+/hwrPDErNdLxYSZUxaYJM6VWaKjOVGIca4aqDAAsR5CBVzmFmKkHmK5yDzyxj1cjhW9DKznQ+DrmjOR/ZrOxhRkAwGKTay0rXde2lJgzluUSYkptqSpFLm1sMVvQpLBtaCW3nNFqFr7NjBnMAGAxggwGy+VAlwSY9FIFm9ihRgoTaobMdOYz0HTlM8z4CjS7D5weNdAwZgYA2jOznzOzT5rZFWb2ZjPbYmavM7PPmNnl9en8dfczySDjq4UBeVRhCDD5mmKoGRpshk7d7CPQ9A0zVGfCoSoDYCzM7ExJPy3pAufcV0s6QdLj64uf7Zw7vz5dvu6+GCOTmI89qKmkDjGEl7LMv14xxtqkGFcj+Zk0YMj4Gan6bhkyfmbI2JmpTgTAeBkAaO1ESSeb2R2S7irpur53AnSWMsQQYMZh9nWMNYFADsGma6hJPSHAgX0nJwszUnkTAYQMM8xgBqCv247cJWaFeqeZXTbz9yXOuUuaP5xz15rZb0j6D0m3SXqPc+49ZvaDkl5kZs+V9P8k/aJz7vZVC5pka1lXpR5LJtRA/1Qhhhay8cphbE3sSQM63y7hhAC0mnXDeBkAE7fHOXfBzOmS2QvNbIekR0u6n6QzJG01sydKeo6kr5L0nyWdJukX1i2Iigw6SRFiUoaXk/as3BGw1O07T/K8JtOSolojxa3YNGGmT4VmSLuZ1K9CQ6tZN6EqM1RlAIzAt0v6jHPuRkkysz+X9E3OuTfVl99uZn8o6Vnr7miyQeacU/d43cs3BbFDTIkBxsftCUEbpRhbc3TZEYJNn0DjY/yM1D/QcMyZdhgzAwAL/Yekh5jZXVW1lj1c0mVmdrpz7nozM0nfI+mKdXc02SCDbmKGmNTtY0NDTG7LH1swSlWtkTZ+DnyHmpICzdiqMzEPoOkDVRkAJXPOfdjM3irpY5IOS/pnSZdI+iszu4ckk3S5pIvW3RdBpiBDpmQdYiohJnWACSXW40oRmHIJNZK/YLN5b9wJAaRhgWYMEwGEbDWjKgMAx3POPU/S8+bO/rau98Ng/5HyMdA/9rE4UoWYk/bcPtoQE1PzPKZ6PlNNGHB0+R4nDhgyIcAQfSYEYCKA9UIM/ue4MgBAkMESUxgPQ4AJi2AzPNj0CTQ+DqjZ1YF9J/cOND5ctX+nt0Cz+8DpQQINYQYA/Jt0kPE1YHRsYldhUoUYxJUy1Eh5BZuuYgeavtM1pwwzUv7VGaZlBgC/RhdkTt50R5D7LfVYMl2NvZWMKkweUldrpLTBZkig6XybgYGmqz7VGd+tZj6rM775DjNUZQBM2eiCDPobc4ghwORtqsGmT6CJPX5m6tWZUK1mAIDhCDIj1GcjZ+whBmVJHWqkuMEmVqBJUZ3pagoTAVCVAQA/RhlkukyhGWKcTJfpSPsckG6drlO3jlXqDWH4kUO1RooTbHIfP8NEAHmHGQINgKkZZZAJJfU4mS7HiOgaZkIcsXzpsiId72NsB4JEJZdwGirQDJnhrPNtIoUZaVytZr6EmsmMQANgKkYbZMZalSHMAPmEGSlMq2TMVs++YWbq42Z8CTWTWRNoCDUAxmy0QSaULlWZHMJMV4QZlCK3MOM70MSa1UwaNhFAV2MZN1NCmDl6/4QaACM16iATqioTqsUsl/EyYwoztJeNW+pxM/MIM+30CTNSfuNmSgozR5dDqAEwIqMOMjnoUpXpImSLmVSFmViBhsoMhsotzKQ4RtIswkw7Uw0zR5dHqAFQOILMjNKqMqHDjBSvOkOYwVA5hRnJX3Um5uB/iTDTR8lh5uhyCTQACjT6INOlvSyUrlWZXMbLNEoPM7SXTUeOYcZHoCHMHC+3cTNjCDMSVRoAZRl9kOkqh6pMV23DzJDjy5QeZjAduYUZyU+giTmTmVRGmJHyGjczljBzdB0INQAyN4kgM/aqTBeEGUxBjmFGGt5uFnPwvxQ/zIyh1cxnmMkJoQZAjiYRZLoKVZXJpcWshDDjG+1l00OYOaaUMCONY9yMrzCTQ1VmEUINgFxMJsjkUJXpI7fxMlKcMENVBj7kNj1zI8WsZlMJM7mMmxl7mGkQaACkNJkg01UuVZmuYoyXkcqszFCVCevEG7+YehWWyjHQDAkzsQf/S8PCzFTHzUwlzEhUaQCkQZDxJJcWsy5yDzNUZcpw4o1fPBpicg4zUr7tZn2UFGakcicBkIZVZ6YUZhoEGgCxTCrIdG0v61KV6WoM42UkwsyUzQaY+fNzllN1JtVMZoSZ7ggz3RFmAIQ2qSATWi7TMUvdw8zQCQBKaTWjvWy4ZQFm/jq5yyXQlDYts5QmzJQ+boYwAwD+TS7ITKUqI3Uf/J9rdYaqTB7aBJj565dgDGGmjyFVGSl+mJHyqM4QZrojzAAIZXJBJrSuVZlcxss0CDNYpG8o6Rp+UsmlOtNXihYziTDTx5TDDIEGgG+TDDI5VWX6CD0lc65hxhfay9rzFURKCDNS2kBT4ngZiTDTx1TDjER1BoBfkwwyoYWuynQ1ljBDVSaeEJWUUsKMlK7drOQwk+JYM6kPnkmY6YcwA8AXgkxLoasyuY2XkfIMMwgrdCtYaWEmRaApNcxI8Q+cKaU/eCZhph/CDAAfJhtkuraXdRVjBrPQ42Wk/MKMr6oM7WUbxRzLUlKYkcofP9PF1MKM5Kc6k0uYKS3QEGYADDXZINNH16rMGFrMpPGGGaQbjF9amJHitpulnJI5dZgpddxMDmFGKi/QEGYADDHpIBO6KtNHji1mkp9jzSAvqcNE6uX3EbM6M9UwI5U7CUAuYUYqK9AQZgD0Nekg00foqkwfscKMlM+BM31UZabaXpbTlMi5rEdXsQINYaa71AfPzCnMSOWMnyHMAOijf1PySJy79fogPx5DbN12W+cf4iPbD3f64T+040jvjYxD24dt5BzcYUmORj51uYaGZr0O3+Nuideku5P23J59IN5yi+u9A2Hz3uGtpZtv2dR758mmvSf2Hgt4YN/Jvdt1r9u3bdBOqKv27+w9QczuA6d77xZowsx5W672er++7Tp0is7bfGvq1QBG6fY7Txy0oyVXVGR6iFGVCT1eZqgcxs1QlWknpwrMKiWs4yKhqzNDqzLSNCszUtpWs9wqM1IZ7WZUZgB0QZBRnmNl+ojZYiaNJ8yMVSkBZlZp6zsrZKDxEWaGIMz0k2OYkfIPNLsOnUKgAdDK6ILMyZsORVlOrlWZPmEm1ZgZiUkAQik5EJS87lK42c1SjpeR8ggzJY6byTXMSGUEGgBYZXRBRsq3DzjnFrOSw8zQqsyY2stKrMIsUvpjyPXYM6WHGanM6kzOYUbKO9AQZgCsMsog00ef9rI+gzlznMWsUXKYmbqxBJhZY3g8vgNN6vEyUhVmhgYawkw3sSakIcwAKM1og0yfqkyssTIxDpTZN8wMMTTMDDHlsTJj2OBfZiwBzWegySHMSH7CzNTGzZQSZnINNAAwb7RBJpa+U2x2FSvMDJ0AYAiqMt2NYSO/jbE8Tl+BZixhRpreJAAlhBkpv+oMVRkAi4w6yMSqysRqMSshzNBihlDGUp2R/EwIkFOYSdlqVmKYKUVuYQYA5o06yPSVa4uZNI0wg3bGslHf1VgCzZjCjJS21SxVmOmrlKqMlFeYoSoDYN7og0ysGcz6tpjlHGZKM2SczJhmLpsKwkxlTGFG6l+dSRFmptBiJuUVZgBg1uiDTF+xWsykfMNMqqoM7WXrjWEj3ocxVGdyCjM5tZr1CTSEmXByCTNUZQDMmkSQyfW4MrPGGGaAWEoPNLmEGan86kxpYaYkuYQZAGhMIsj0FbMqI40vzKSoykx5GmaUXakaa5hJMRFASWGmpKqMRJgBkJfJBJm+VZnYYaaPnMfMMPDfv5I31mMouTrjK8zk1GomEWbWKTHMpAw0tJcBaEwmyEjjbTGT+oWZLlK0mDFWBkOUGmhyOnCmlE+rWSlhpq/SwoxEdQZAepMKMn2V0GImdQ8zJbSY9UF7GWaVGGhyDDM5TARQQphJNV5mamGGqgwAaYJBppQWM8JMJWZVpoQpmEvbIM9JE2hKeQ5zCzNSHtWZMYeZIVUZaXphBgAmF2RiSxFmuprCMWaAWaUEmlzDjK/qTO/bEmaWmlKYoSoDYJJBJmZVZqgcZzIrpcUMWKeEKk2OYUbyU50Z0mpGmFluSmEGwLRNMsgMkWIWs1hhpouYg//7tJeNcZxMzhvbY5BzqMk5zKSszow5zAw1lTBDVQaYtskGmdgzmMWeklnKc7wMVRmUIMdAk2uYkdJWZ0oIM30MrcpIVZjhWDMAxmyyQWaIUlrMpPGEGaZiRgq5VWnGHmakcYaZVC1mDcIMgLGadJAZUpUppcVMCh9mctWnvSzXmcty2ZCeslwCTe5hJlWrGWFmtTGHGdrLgOmadJBJYYxhhqoMpiSHKk3OYUZK12oWO8x0lWq8TCPVuBkACGXyQSZ2VUYqJ8x0EXPwP5CLlKGmhDCTojoTM8zEGvzvqyojxQ8zVGUAhDT5IDPUmMNM6PEyMQb+j2H2shzambBeikDjM8zkXp3pdP3Mw0wfhBkAOB5BRvFnMGuMMcx01TXM0F6G3MWu0vgKM1Le1ZkxhZnULWbSONvMqMoA00OQqaVoMZPSTMsshQsztJgBx8QKNCWEGWl4dWbqYcZnVUaKOz0zVRkAIRBkCte3KiPlE2aoyixHW9k4xKjS+A4zuVZncg4zXeUQZqR41ZkYYYaqDDAtBJkZpVZlhoSZUHIaL9N1nEyuUzBjHEoJM1K+1Zlcw0yJg/8bYwozAKaDIONRiWEmp/EyXUypKoPxKS3M5DgRwNTDTAhjHDcDYNySBhkze62ZfcHMrlhyuZnZ75jZlWb2CTP7utDrlGrgvzS+MJNTVaZEpbWVuRtuPHrCeiWFGSnPiQDGFGa6ClGVkeKEmdBVGdrLgLyZ2Vea2eUzpy+a2c+a2cVmdu3M+Y9ad1+pKzKvk/SIFZc/UtL969OFkl657g5PtrSDzYdUZSTCTChjmIY5Z/PhhUDTTolhJrfqzFjCTC4tZtI4wgyAfDnn/tU5d75z7nxJXy/pS5LeXl/8suYy59yl6+4raZBxzr1f0s0rrvJoSW9wlQ9J2m5ma79hz9t866D1GlqVIcxs1CXMdKnK0F6Wh1WBhUCzXmlhRsqvOkOY8a/0NjOqMkAxHi7pKufc5/rcOHVFZp0zJc2mimvq8zYwswvN7DIzu+ymm/KoAAxFmGmnS5jpUpXJYcD/4XvcLfUqrNU2pBBmVis1zORUnck1zHSVy3gZKfz0zFRlgNHa2WyX16cLV1z38ZLePPP3M+rhJK81sx3rFhRnjsnAnHOXSLpEks5/0GYnVVWZIXtkztty9aAv2XO3Xj/4B+CcU/cM2tt2xrZ9vX4Ut267rfWexCPbD7f+cT+040jrjYdD29tvmDRhps3GTxNm2myANWEm1IZgG/NhJrdxM3ave0haHVSa62C1E2/8YhHhNZYtt7jgVdfNt2wK3v56YN/JnXYQ9XHV/p3JjkmWs12HThncoQGMxR13nhBzR8ke59wF665kZpslfbek59RnvVLSCyS5+t/flPS0VfeRe0XmWkmzaeKs+rwoUreYSVRmcqnO5FChkapg05xyMh9W7F73OHpCeinDeExdqzJ95FqVCdViJoVtM6MqA0zaIyV9zDl3gyQ5525wzt3pnDsi6dWSHrzuDnIPMu+Q9OR69rKHSNrnnGudDnzsicklzAwJNGds29cr0OQQZqRugebgDmsdaA6edkKvQJNTqMkJ4WWaQh5rZixizGIGAAV6gmbayubGwT9G0sJZjWelnn75zZL+UdJXmtk1ZvYjZnaRmV1UX+VSSZ+WdKWqZPYTKdYzhzAjpanObN12W+tAc2T74daB5tCOI0GnZw5VnWnkEmpyCzNAafqMlYmBqoyn+2bQP5AlM9sq6Tsk/fnM2S81s11m9glJD5P0c+vuJ+kYGefcE9Zc7iT95JBlDB0r44uPMTPStMfNSMfCTJsWklBjZ+bNhpkULTxNmMlt/AwwVIxxMn1s2ntiVgcInsV4GQAlcM4dkHT3ufOe1PV+8twVlSEfB8r0WZmZcquZlFd1ZlbKSg3VGSBfY2gvK31KZgDjM4kg42vWkpzCjJSu1aytGGEmh7Ezy6QINYQZLBOyWpjTOJkYA/5j6TvDUMgWs1AY9A+gj0kEGZ98hRnGzRyvz7gZqXt1JnagkeKGGsJMuWgPTCPXcTI5KrEqk0N7OYAwJvPt7XMueR9hRsqr1ayPUqszUtx2s3kxQg1hBgirzzTMMdvLqMoAmILJBJlc5VKdyXHcTI7tZiWFmhyPNwOgmyEHsAsVZkqsygAYJ4JMT76qMo0cwoyU17gZKV67WRchAo0ULtQQZhBDqHEybWYdnDemcTLwg/YyYJwmFWR8tpdJeYeZ2K1mXcNMn0DTVajqzNHbBAo0kv9QQ5gBVuszTib39jKpvKoM7WUAuphUkAkhRJjJoToTehIAaRzVGSlsoJE2hpohwYYwgxTHOcJwQ9rLAGDMJhdkfFdlQsklzOTWaib1CzShqzNS+EDTGBJoCDPA9FCVqe+X9jJgdCYXZELwXZVp+KrOlNBq1kffQNPW0EATOtQQZpCbnI4nk7M+7WVDqzIlzmIWAmEGGJdJBpkQVZlQYUbKpzrTVehxM42Q1RmpX7vZ0dtGajvrihnN8sWxZPzoM+A/1jiZMSlxBjPCDDAekwwyoRBmjhd63Ewj13azo7ePEGj6IMygBH1mLsPxSqrKhB70T5gBxmGyQSbUWJnQYSZ1q1mscTNjazc7evvAM531QZiZFgb85yFFe1koJVZlJMIMMAaTDTIlK7k600XfMCPFaTfLMdAQZoD4SmovoyoztwzCDFC0SQeZEqsyDZ/Vmb5ihZlcqzOSv0DjM9QQZpBS6QP++4yTKU2IMFNqVUYizAAlG/83diIxwozkpzqT+7gZKW6g6VqdkYYHGslvlYZJAIB+A/5zl2t7WekIM0CZJh9kQh5XprQwk/O4mUbsdrOucgw0fRBmgPD6jJPxpZSqTIz2sqPLIswAxZl8kAktZpgptTrTVe7VGSmvQEOYwRjkPnNZzHEyVGXCIcwAZSHIKGxVJraphBmJQNMFYaYsoY8lE3rmMsbJlIGqzJLlEWaAYkzj2zqxWFWZho/qTAnjZho5z27W8Blo+oYawgyQn5TtZVJZs5jFRJgBykCQiSR2mJGGV2dKGTcjpZndLFWgkZQkzBBoUKKYA/5pL6uUPIPZLMIMkD+CTC1Ge9l5W65OUp0ZqrTqzFQCTd/qTN8ZzSSqM8AYlVCVid1ednS5h04h0AAZI8gkQKtZO33DjDS83ay0QNMHYQYhMU6mvdTtZSGMpSrTIMwAeSLIzIg56L/E6kyqMJOiOiN1Hz8jpQs0hBl0FXrAfwi5z1wWm8/2MqoyLZZPmAGyQ5CZE3sGs9KqMynGzUhltZtJaQINYQZIL+Y4Gd98h5mxVWUkwgyQG4LMAinCTOwKjY9A01ffMCOlDTS9btcjzEiKHmaAMYs54L+vvu1lOQ/6HyvCDJAPgswSqY4tU1KgSVWdkdIEmlLCTB9UZfIT+lgyMTBOpjy5V2VSt5cByMv0vqU7OG/zrckDTaxQU2K7mRQ/0AxpNeujT6sZVRkgrb7tZWMc9D9WVGWAPBBkWkgVZo4uP1KgKXX8jHQs0PQJNU2g6RJqYo6bkbpXZ/pOy9wHVZlyMeB/HHy3l5Uw8D8HhBkgPYJMSymrM0fXoaBA09fQQCPFq9KMudUM8CWX9rK+42T6tpeVPOjfN9rLAIRCkOkodZiR4rWdpRo/I5UTaHIOMzGrMgD8YNB/WajKAGmxy6iH8zbfms2XVxNmQu6hOnfr9b33qDVhpm+rQhNmhvxIN2GmzwZCE2ZW7V09tONIr722h7b320t8cIe1bq85eNoJUfaIH77H3UYxOB1A5ar9OwftjJq3+8Dpg49llqtdh07JYicnsMqROzeNchweFZmecmg1mxW6QpOy3UzKo0KzSopJAEKhKgP4EXvQP1UZAFNDkBkopzAjhW87S9luJh0LNClmOgvVajbk4JmtrscsZsWKUeWKMeDfd1Ww74D/2ONkUvAZZnIe9J/bOJlcujSAqSnn2zljuVVnGqEDTV8+Ao2UZurm3MbNhKrM9KnKMHsZgFV8D/rPDWEGiI8g41GOYUYKF2hSt5s1Yk/d3GYigNiTAKwTqypDmAE2SnFMmalUZQCAIONZrmFGCtd2lrrdrBF7HE0uYSZUi9mQ48oQaDA2JbWXTUVu7WUSVRkgNr6ZA8i11WzWFAJNjLazXCYByPEYMwQazCr9eDIpjLEqM/b2MokwA8REkAko9zAjhQs0ffkMNFKccTS5TALQJszEqsrMItDkL8aAf9/6DvhPgYNjAkAYBJnASqjOSP4DjY/xMzm1nbUJM7m0mq2TahYzwgwQ1xSqMjm2l0lUZYBYCDKRNIEm91CTW6CR8mk789Vq1kfbMJNji9ksqjPdjelAo7m0l/WVYpzMGA9gNxWEGSA8gkwCs6Em12ATItAMlUOVZixhpktVJsQBMgk0SCnFOJlU7WW5VmWmgjADhEWQyUDOwcZnoPFRnWmkDDQ+xs3EmgRglRwOlEmgAdYbY1VmCu1lAMIjyGQox2CTe6BJcYDNnCcB8N1iFqIqM4tAkx4D/tspcRpmn1UZdEdVBginvG/kCcop2OQaaCR/VZou42hSTgKwTohZzEIj0Ixf6eNkhhjD7GW5DvrPHWEGCIMgU6Acgo3vQONT7CpNqnEzbdrMfIaZ0FWZWQQahFbS8WSk4e1lVGUAjBFBZgRSBhtfgcZ3dabhM9CsknOYKRmBBjkqsb3MpxwH/ZcwToaqDODftL+NRypFsCkl0AwJNW3CTIpJAHyMl8mxKjNr6oFmTFMwS7SX9TXGqsyU2sskwgzgG0FmAmIGG9+BJrdQ42vczDq+x82UOF5mkakHmtAY8I+2cqzKAJgegswExQg1IQ6qmUvrmY9xM74nAfA1XqaNVFWZWYQZ+DBknMyQ9rKUg/5zrMr4UkJ7mURVBvCJIDNxIQON74NqSnm1nuU2bsbHg1iS6QAAIABJREFUeJkSqjINqjOYIh/HlPEVZnxVZabWXiYRZgBfCDKQVG6gSV2lyXXczDK+WsxyqMo0CDRlmvI4GQCAHwQZbFBaoJHChZq2VZqcxs3EbDHLDYHGjxLHyZQo5aB/abxVmVLayySqMoAPBBksVGKgkcK2nq1SWphZp7SqzCwCzfQMGfCfapwMIBFmgKH4FsZKpQcan6GmTZjJZRKAmFMy56oJNGMJNWObghnp5VaVAYCuCDJILlSYacQMM1I+kwC0CTPrAs26MHP7zpOyrczMKj3UxFz3El7PsUjdXpabKQ76l6jKAEMQZNBKjOPPhOSzOpNTmFkXaHyMmSm5zWyR0kJNKevZVekVP+SrpHEyAIYhyKC10sOM5K860+dgmvPahJk22oSZoa1mB087YTTVmVm5h5rY61Xa69cY6wQWAIDVCDLoJHSYiSHEZACLxJoAQMqn1Uw6FmjmT7nLLdSMOcRQjQEA+ECQQVZCTgAwy0ermY8WM8lvmMmlOrNISeEmdagZc4gBAMAXggw6i1GViRFmpOHVmdzCjJRXdWadEqo3sUPN2ENMbtUYH9OVA0Mx4B/ohyCDXmKFmRKqM7mGmVwmAugj13ATOtQQYhCbrymYASAFggyyV0J1JscwI+UzEYAPuVVvfIaaFG1suYRDAAD6Isigt5gD/2NWZ/oqNcxI+bSa9ZFDuBkSalKMw0nxHIV6fzBjWR44KCaAFAgyGCT2LGYltJqtkyrMxKzOxKrSLJMy2HQJNYQYAMAUmdl2M3urmf2Lme02s280s4vN7Fozu7w+PWrd/RBkMFiKMJNrdabt8WV8hZlcqzNHr5tZsIltVaiZSogButh94PTUq5AMA/4xMS+X9NfOua+S9CBJu+vzX+acO78+XbruTggyKFau1ZmYYUYKV51Zp09LT+pQk8uYmimFGKoxSGHXwbNTrwKAJcxsm6RvlfQaSXLOHXLO7e1zXyf6XDFM13mbb02yN6kJM7n9aJ1z6p5WPeNnbNvnZdagJsxs2tv+I31oxxFtvmX5vowmzGxe8dVycIdpyy2u9TI33HZuA3fLzXf2up+umg36k/bcHmV5qVGFAQDoTuu0jTDQTjO7bObvS5xzl8z8fT9JN0r6QzN7kKSPSvqZ+rJnmNmTJV0m6ZnOuVtWLYiKDLyJ3WK2YdmBqzMp28zaVGUaubearbyfyNWa1LOexZD68VGNAYBJ2uOcu2DmdMnc5SdK+jpJr3TOfa2kA5J+UdIrJZ0j6XxJ10v6zXULIsjAq9RhJmSgKSnMhDjmzNCJALqIObYm9cZ+KKkfV4wQM/Q9N/RgmG12BIR0YN/JSZcPAD1dI+ka59yH67/fKunrnHM3OOfudM4dkfRqSQ9ed0cEGYwOYaaSsjrje0rc0MFmbNWZMT0WYEoY8I8pcM59XtLVZvaV9VkPl/QpM5ud7eMxkq5Yd18EGXiXsipzdB0CVmdSTs3cJ8yEqM60ESLQHL3vQKFmDAEgh8dASxkAYI2fkvRHZvYJVa1kL5b0UjPbVZ/3MEk/t+5OGOyPIFIN/j9uPTKZDKDt4H9p/QQAW7fd1rml5Mj2w9EnAmjMhpm+EwOsvH/PkwaUPBlADiEGAIB1nHOXS7pg7uwndb0fKjIIJofKTMN3dSZki5nkvzIjpa3ONEK1nm1YhqdqTWntZrmsK9UYAEAsBBlMhu92sxzCTN9A04WPiQAWCR1oJHkLNLnLZR0JMWXyMQU8AKRAkEFQOVVlGmMKM1K86sza6/ScASpmlaavnKszua5XaKFDMLpr2z67zu4Dp6+/0ojl0JYNlIIgg+ByDTO+Ak0uYSZ0dcbXNM2rhA41Yws0Oa0L1RjkJvXYSADhEWQQRY5hRgp/IE1f2oQZKa/qjK9QE8IY2s1yWAcAAFIiyGDyfFRnQldlusilOnP0ugNDTehj0/SVqjqTW1VIKrMaM/RgmACA9AgyiCbXqkwj9zDTtirTyKU6s+H6GbaelRRocgswUpkhBgAwDgQZRFVCmBkSaHIMMzGqM13l2HqWe7tZjiEGQBgM+AfaIcggutzDjDSsOpNbmJH6V2fa6tJqdtxtM6rS5FqdyTXEpKrGMGMZAEBKHGTM7BFm9q9mdqWZ/eKCy59qZjea2eX16UdTrCemyfdxZ9aJEWa6BpoQB9FceluPVZqhG7o+Ao0vhBgAABZLFmTM7ARJvyfpkZIeIOkJZvaABVf9U+fc+fXpD6KuJIIpoSrT6BNm+lRlujpj274sqzPSsUCTQ6gZInV1JtcQAwBADtYGGTP7KTPbEWDZD5Z0pXPu0865Q5L+RNKjAywHmSLMHK/PTGa5VmcaQwKNlD7QpGo3yznEUI0BAOSgTUXmXpI+YmZvqVvBfDUnnylpduvwmvq8ed9nZp8ws7ea2cKjW5nZhWZ2mZlddtNN/TeYEB9h5njnnLqnc6BpqjMxJgMoOdAMEWvjPcfplWcRYlCSkg+KyYB/YL21QcY59yuS7i/pNZKeKunfzezFZnZO4HWTpHdKuq9z7mskvVfS65es4yXOuQuccxfc/e7MX1Ca8zbfWkyg6TNu5tyt10c9zswUAk1fKQeJtwknOQcYKY8Q4+M19HEMmSHvYZ8O7Ds59SpscNX+nV7uZ/eB073cD4Bxa7XV75xzkj5fnw5L2iHprWb20gHLvlbS7K6Ss+rzZpd7k3Pu9vrPP5D09QOWh8yVFmi6ilWdaZQQaPoaeoDNvkJN0Zx7FUbKI8QgnOv2bUu9CgDQWZsxMj9jZh+V9FJJ/yDpPOfc01WFiu8bsOyPSLq/md3PzDZLerykd8wte3aXzHdL2j1geShEE2hyDzUlVGekvANNqnazlGFmVgkBRiLEAADydGKL65wm6Xudc5+bPdM5d8TMvqvvgp1zh83sGZLeLekESa91zn3SzH5V0mXOuXdI+mkz+25VVaCbVbW2YUJmw0yu/cLnbbm6cx/2uVuv79w60YSZvq0bTZjpsue1CTNd2leaMLNpb5uvl0oTZjbf0q819NB2afPebrc5uMO05RbXa3lDlBBcZhFiAAC5Wrul4Zx73orLBlVInHOXSrp07rznzvz/OZKeM2QZGI+cQ01TmekSaJrKTJ9AM6QPfUigkdqHmiPbD3cKM1IVaIaEGal7oOnj4GknaMvNd4ZfUGKEGABAzhgZjyLl2n5WwtiZRoyZznJvN8ulxSxHOT6+lJM1ACnkttMOyE233aVAhnKr1MSuzjRSVWnaVGhiV2ek9u1mqVrMcpZjiAEAYB4VGYxKTpWaWNWZhs8qTRdtKzQ5V2f67ukf2wb/0IN/lsDH1MsAgDwQZDBaOYSamDObNUoINF3FOPbM1MPMWB4HAGA6CDKYhNShJnZ1RjoWaFJM3bxOimPPtDHVMRglhJipvjYYruuskgDKQZDB5KQKNSmqM43YgSbH6kzICQBKCAKLTKGVDOXqOmYQwPQQZDBpKUJNiupMY2iVputMZ6GrM10DDWGmQoBBSEMmHsHxcpjEBsgVQQaoxQw1KaszjVhVmpDVGal7u1mMqZlzRYABAIwJQQZYIFaoSVmdafiq0qyTU3Um1MxVuYYEAgza6DL9OgDkgCADrBE60PQNM74rNNKwKk2bQBOjOtM20ISaySynwDCWAOOrOuYrwIaedAIA0A5BBmgpZKDp02rWCBFqhlRpUldnpPYbmiGnZU5pLAEG7bQ5KC0AjBFBBugodKAZImSo6aKk6kyIMJMqRBBgAABTQpABesqxOjPLd6jpU6UppTpTepghwADjxsxlwGInpl4BoGRNmAnxI3Pelqu9HchtNsz4ODZDE2baTLPahJlVA4mbMLOuRebI9sPatLf719ahHUe0+ZbV+20ObZc27119Pwd3mLbc4lov9+BpJ2jLzXe2vn4XBBegvV0Hz/aygwgolR3W2t/BEhFkAA9CBZrmh9fnkal9hprZ6sy6UHPGtn1rZ0Xauu22VmFGUudAkyrM+Da1AFPiGCUAQBzji2ZAQqHGz4Tak+iz/axN21nbsTNt9J2mee11PLeZ+QoetI9hinxUkAGMF0EGCCBEoPE1dmYZX6GmzRganxMBdA00pYUZAowfoY4dBABIhyADBFRSdWbW0FDTtjqzTqjqjK/jgIRseyLAAACwGkEGCCxkdSb3UOOr1Sz0zGaL+N6D3zaUEGCOYXxMudpMBIJumLkMOB5BBogk5PiZnEONj1YzyX91JrcWMwIMAADdMGsZEFnoKZtn+ZztbF6X2c/aTNfse5pmaf3MZjnMZEZ4QU6u27et1Y4FAMgBFRkgkVAVmg3LiFStaVulybE6E7sy0wQXKjCr0VYGAFiHIAMkFjrMHF1OhFDTJtD4nKa57diZdVKFGSxGiEEIISvUANIgyAAZiFGd2bC8mVATItjkVp3JMcwgHp8TN/ia8Q4AMBxBBshIzDCzYbmEGUmEmdgO7rCFp6nwOcsepoGZy4CNCDJAZlKGGd+BJscws27j8dCOI2sDDWGmm5SBhQNhYkxS/T4AuSLIANhg7GFG8lOdObR9/UbylCoMy8LKmB7/FNrKmLEMQEkIMkCGUu91i3FMmnklhhlpetWZksJKztUY2soAYDiCDJCpMYWZtgfQbBNmfCLMLDaF6kpXU6jGAEBpOCAmgKWaMJPTtKVnbNu38qCZUlWVWXfQzEabg2c2G7GrDp7ZhJlVB88ccuDMtqYcNmblXI3xrW0VEgDGhiADZOy8zbdmMUvNeVuuHhxmzt16vXYfOH3t9c45dY+u2r9z5XV8hxmpCjSrwoxUBZpVYUaqNqDXhRlMG21lAOAHrWVA5lK3mDV8tJr5bDHzOV6mEbPVDOH4fv5pK+sudpsogGkiyABoLeYkAIQZIC5mLANQGoIMUIBcqjLS8DDTtiojpQ0zvo43Q6CJK/dqDG1lAOAPQQYoRG5hZkig6RJmfOkzIJrqDHLHQH8AU0aQAdBbjFYzX1UZqdroo9VsvHKvxgAA/CLIAAXJqSrT6BtmUrSYNUK1mq1DmJk22soAwC+CDIDBphBmpPUbol3GzRBq/KIaAwDTQ5ABCpNjVUYqr81MSttqJm0MNetOWG6qzw/jYwBMHUEGKNCYwkzXgf8hjk+RMsy0RegpW+5tZUy9DKBEBBmgUFMOM+v02ShLNUVzCFMKPSEeA21lw3AwTACxEGSAguUaZkLz3WLWSDlFcypjDDcAgGkgyACFyzHM5NJi1jfMlNBqFlIpgaaUakyItjLGxwAAQQYYhfM235ploOkqlzAjlTFuJjSqNACAnBFkgBHJKczEmMVMyi/M5DpuZqjcAk0p1RgM43v8HIBxIcgAI5NTmOkj1IbLGdv2ZddqNn8qAVWabnKfrUxixjIA8ZnZZ81sl5ldbmaX1eddbGbX1uddbmaPWnc/BBlghHIJMzEOlCl1myUpp1azeaUFm1ShZuohivExAEbiYc65851zF8yc97L6vPOdc5euuwOCDDBSpYeZrsYSZmaVVLUpvUqT83MLAFiMIAOMWMmTAPRpMcs1zPhsL8o92ISu0pQUlkpoKwOAAHaa2WUzpwsXXMdJeo+ZfXTu8meY2SfM7LVmtmPdgk70tsoAsnXe5lu169Ap6Za/5WrtOnh2lGWdc+oeXbV/Z6vrNmHmun3bOi2jCTMH9p3c+jZHth/Wpr3+v3IXhZnNt+Sxj6oJHZv3pl2PdXIMhADg06Y7o34X75lrF1vkm51z15rZPSW918z+RdIrJb1AVch5gaTflPS0VXeSx68dgOBKrMz0Hfjf9cjipbSatZVbS5qvKk1J1ZhQch8f0/WzB2CanHPX1v9+QdLbJT3YOXeDc+5O59wRSa+W9OB190OQASYkZZiJNfC/r1JbzdrKJdj0DTXBWtUCPRe0lQHAYma21cxObf4v6f+TdIWZnT5ztcdIumLdfdFaBkxME2ZStprF0KXFrBG71WxWiLazVXJoSSul9WzMmHoZQAL3kvR2M5OqLPLHzrm/NrM3mtn5qlrLPivpx9fdEUEGmKgU42b6jpU5d+v12n3g9PVXnNMnzEjVxl3XMCNVgaZLmJmVOthIx8JNqkAjLQ41tJQBwHg45z4t6UELzn9S1/uitQyYsBSzmsVuMevbsx/zAJqLNC1oKVrRUrafxQwttJUBQNkIMgCiB5oUYabUQNNIFWhSmA0zpY2NCSn3gf4AEButZQCOijl+pgkzXVvNmjDTt9VMUu92MynO+JlVZsNMjPazQzuOJJnOudR2MqoxABAPFRkAx2kqNDGqNClmMxtDhUaKV6UpsXqxytgeDwBMFUEGwEoxAk2qqZmHBpo+QgaakKGGjf+0QrSVMWNZWUo8FhgQGkEGQCuhqzQpjzPTN9D0rc5IYQKNFDbUjCHMhHwMU28r42CYAGIjyADoLFSgOW/L1b0CzblbryfQLBAi0JQcZkpedwDA8QgyAHoLGWj68BFmpPEGGl+hhkBwvJDVGGYrA4DFCDIABgsRaFKHGWl8gUbyV6UpLcyUtr4AgPUIMgC88R1oUreaNcYcaIaEGsJBpcSxMSUM9Pf5GQYwTgQZAN6NsTojjTPQSMOqNCWEmRLWcRnaygBgOYIMgCBCVGf68F2dkdIGmtBVmj5yDgqh163EagwAjAVBBkBQOYQZ6VigKb3lTAo/MUAfOYcZAMA4EWQABJfDuJlZBJrVxhJmclufrkpqK+MYMgBSIMgAiCKXVrNZvqs0Ywo0YwkzIdFWBgBpEWQARJVbmGkQaI5XcpjJYR1yVcKMZQDQBkEGQHS5tZrN8lmlGRJohvAZaPrOaJYySMRYduhqTEltZQCQCkEGQBI5tprNSxlohlZnJP+BpiuqIsjNroNnp14FAB4RZAAklXuYkfxVaVIHmqGhpoQwM4ZqDACgHYIMgORKCDONkgONNLxKk3OYGUsFiLYyAGjnxNQrAADSsTCz69Apw++rDjMh20hmw8zuA6f3uo8mzFy1f2fr2zRh5rp923ots9FsLB/Yd3Ln2x7Zflib9nb7+Ti044g238K+s9QY6A9gTPhVAZCVkqozjaFVmr4VGh/6Vmhyq8zEqsbQVgYA+aAiAyA7522+1UtlRtoYZkIP9B1apelaofFVnZGqQNO1OkNlxr8S28o4GCaAVPg1AZAl37OaScemao5RqRlSpUk5fqarHCozVGMAYJqoyADIms/qzIb7jVSp6VulSTV+ps/YmZSVmbEM8AcAdEdFBkD2QlRnNtx/pEpNnypNyimbu8ihMhPSGKoxDPQHMDYEGQDFCBlmji4jQqiJGWiGyD3MjK2lrMTxMQCQEkEGQFFihJmjywocamIEmqHVmdzDDABgupIGGTN7hJn9q5ldaWa/uODyk8zsT+vLP2xm942/lgByE7rVbOEyA4aavoGmiyGBpk+Y6RpouoaZsVVjsNHQg84CmIZkQcbMTpD0e5IeKekBkp5gZg+Yu9qPSLrFOfcVkl4m6dfjriWAnMUOM0eXGyjUdA00McfPxJjRrG04GWMFh7YyAOguZUXmwZKudM592jl3SNKfSHr03HUeLen19f/fKunhZmYR1xFA5lKFmaPLDxRouog1fiaHMBMzxIypGhNqoD/HkAGQUsrpl8+UNPvrf42kb1h2HefcYTPbJ+nukjZ8c5rZhZIulKSzzjwh1PoCyFSoKZo7rcOWq71O43zu1ut7HVSzizO27es8VXOMA2cumpp5rAGGSkxcMY4hFULqHTYon90pbbnFpV4N70Yx2N85d4lz7gLn3AV3v/soHhKAAsU62OYyffaO516ZObTjSNTxMGMMMaVVYxgfA6CtlFv910qa3X15Vn3ewuuY2YmStkm6KcraAShKTnssfYWZPht0sVp9cmgz8yV2gJGoxKRQajUGwHIpg8xHJN3fzO5nZpslPV7SO+au8w5JT6n//1hJf+OcG19dDIAXYwwzMcSazSw3qQJMzOettGoMAHSRLMg45w5Leoakd0vaLektzrlPmtmvmtl311d7jaS7m9mVkn5e0nFTNANArnyEmVhVmZhTM6eWIsBI5Qe/GGgrO15OO2iA3CQdUOKcu9Q595+cc+c4515Un/dc59w76v8fdM59v3PuK5xzD3bOfTrl+gLIX24/+oSZ46UKM6kCjJQmxISqxpSopAopgPYYGQ8AgaXaiCLMHFtWygAztkoMbWUAckGQATA6uVVlpOEzmpXQcpNbmEkZYKS0rWQlVmNKeI8DyAtBBsAo5RhmpGHVmdxbzKQ8wkzqACONdzxMidWYktvKcv0eA3JBkAGAyGJvWE0lzOQSYFKHGKoxAKaCIANgtMa4NzPmBl/MDeIhASSHACONtwrTKLEaA2DcCDIARi3XMFNCi5kU9xgzXcNILgFGyifElFiNAYC+CDIARi/XMBPbWMJMTgFGyifEhBSyGhOyysj4GGDcCDIAJiHHjYKSZjHLIczkGGByCjFUYwBMDUEGwGSMLcz0MWTPeqowk1uAkaZRhWmUWo0BMH4EGQCTkmOY6avvRmCKQdt9w0xuAUbKM8RQjTleyW1lANohyACYnNzCTIoWs9jjZaQ8A0BbTRtZyY+hD2YqSyO37yggVwQZAJPEhkJ/UwkzJYSXM7btK7YaQ1sZgKEIMgAmK6cwU1JVRhpvmCkhvDRCB5iSqzG0lQHTQJABMGk5hZkhCDPDlBJepLKrMADgE0EGwOSdt/nWLAJNiXuRSw4zJVVfGrECTOhqDG1ly+XwXQSUgiADALXSNyBSzWJWUpgpMbw0qMK0U+IOAQD9EGQAYEbqMJNqI2zMYabk8CKNr5WMagwAXwgyADCn5DAzZCNxTGGm9PDSSBFgSh7kD2BaCDIAsEDqMDNEqXu8h4aOsYQXaXxVGLRT8vcOkAJBBgCWSLlRMcUWM6l7mBlTeGmkDDClD/JnfAwwLQQZAFih1D2kpbaYSevDzBjDS4MqDAC0R5ABgDVShZmhe5fHFmbGGl6kPFrJSq/GAJgeggwAtFBqZWaIHMLMmKsvjdQBZixKbyub4ncMMBRBBgBaSnHgzJRVGSl9mBmzHKowDWYqA1AiggwAdBQ70BBmxqMJL1N7TmgrAxACQQYAeooZaAgz5co9vIyhGkNbGTBNBBkAGChWoMkhzAzZ6M11Qz6E3MNLTFRjAIRCkAEAT2IEmhz2PA8NM2PduC8xvFCNAZCKmZ1gZv9sZu+q/36dmX3GzC6vT+evuw+CDAB4lmJSgLZ87R2n1axSYnhpxAgxVGPWy/W7AojgZyTtnjvv2c658+vT5evugCADAIE0gcb3hkrqFrPGVMNMyeEFAHJgZmdJ+k5JfzDkfk70szoAgFWaMLPr0Cl+7m/L1dp18Ozetz936/XafeD0wetxzql7dNX+nb1vf8a2fbpu37bB6xHSGAPLGFrKpPLbyqjGIJZNh5223HxnrMXtNLP/v737j73srOsE/v7YdjrEsis6LIylKsbGgE4WDMvq9o9lK2aRbFpAMTVRwIVUNktEs7talkQ3rGa7ZleMxnUZUekqggRRqmIaaCFojISB1A5tl1hISFsLpahAXShO++wf3/ull2+/v2bm3nPOc+7rldx8748z5z5znjnfed738zznnlp6fLK1dnLHNr+Y5CeTPH7H8z9XVT+d5OYk17XWHtrvjQQZgAGtMtBMKcwkOedAM5UwM8fAMibTymBjPdBae9ZeL1bVv0lyf2vtQ1X1nKWXXpPkk0mOJDmZ5KeSvG6/NxJkADo2lTCTnF91ZjtEDBFoNj2wzKUa0zvVGDbYFUmuqqrnJzma5B9V1W+31n5o8fpDVfWbSf7jQTuyRgZgBKscxExlzUwynXUzy+tYdt422VAhZohqTO/TymBTtdZe01p7Smvtm5Jck+SW1toPVdXxJKmqSvKCJB85aF+CDMBIhJndnU3YEFbokWoM7OrNVXU6yekkx5L87EF/wNQygBGdOPLg7C4AkKz+IgDCyflTjQGmprX2viTvW9y/8mz/vIoMAF82tcqMCgtzoxoDqyPIAIxsSlPMktWHGYvLxzenakzPhBhYLUEGYALmHGYSV8oa09yOvWllwDZBBmAiphhmpjTVjGlTjdmfagysniADMFOr+uRamOnX3I63agywTJABmJBVf2orzDAE1Zj9qcbAeggyABOzKWFGoFkvxxeYO0EGYII2IcwkBttzMFQ1ptdpZaoxsD6CDMCGEGY2h2MKbAJBBmCipvxJriuaTdeQx9LamP1N+RyGORBkACZsqlPMtgkzDKHXaWXAegkyABO3aWFGoDl3qjHToRoD6yfIAHRgk8JMojrDo3qsxggxMIwLx24AAOM4cfTunP7iZSvb39O++r7c+ffHV7a/w4SZj33+2Mrer3eqMcCmEWQAOnHiyIM5/aVLVrvPNYSZJCsNNPuZWuVmrGA1teOwyVRjYDiCDEBHeggzyeqrM71YRaCYepVpyGpMj9PKgOEIMgCdEWbmTXWlX6oxMCyL/QFIsp5Pv62lmBf9CUyJIAPQoXV98ivMsJeh+7G3aWWqMTA8QQagU72FGYGmX0IMMEWCDACPsa6BpDDTH312MNUYGIcgA9CxdQ6gThy921SzDTdGX/VWjRFiYDyCDAD7EmY2kz4Cpk6QAejcEJ8ICzObZay+UY0BzoYgAzADPYcZgWZa9AfQC0EGgENzEYB5G7MfVGOAsyXIAMzEUAMrFwGYJyEG6I0gAzAjQ35KLMzMh+N+dlRjYBoEGQDOmXUz/Rv7WPdWjRFiYDoEGYCZGXqgZd1MvxxjoGcXjt0AAPq3HWZOf/Gyle53t4H2nX9/fKXvsammEGJUY4DzIcgAzNCJIw/m9JcuGf59j9698jCz034DcCHncKYQYoDh1JmWix94aOxmrJwgAzBTcw4ze9lrgC7gTI9qDHC+BBkAVm7MMLMbAedRU6jG9BZigGkSZABmbKyqTDK9MLObTZumNoUQ0yPVGJgmQQaAtVnXRQCGMLefHTUcAAAO4UlEQVQqzlRCTG/VGCEGpkuQAZi5MasyX25DB9WZw9oZCHoINlMJMQCrJMgAbABhZn2mHmymFGJUY4BVEmQAGMxcw8yyKQUbIQaYM0EGYENMoSqTbEaYWTZWsJlSiOmRagxMnyADsEGmFGaSPi8CcL6GCDZTCzGqMcA6CDIAjGbTqjO7WXWwmVqI6ZFqDPRBkAHYMFOpymwTZr7S+QSbKYaY3qoxQgz0Q5ABYHTCzN4OG2ymGGIA1kmQAdhAU6vKJJu9buZs7BZsphpiVGOAdfqqMd60qr62qt5dVX+1+PmEPbZ7uKpuXdxuHLqdAHM21UFbb4PfsQkxwKYaJcgkuS7Jza21y5PcvHi8my+01p6xuF01XPMAGJNBMEObarAH9jZWkLk6yQ2L+zckecFI7QDYaFMevJ04erdA0yn9BgxhrDUyT2qtbdfCP5nkSXtsd7SqTiU5k+T61tof7LZRVV2b5NokecqlF6y6rQCzNsX1Mst2DoqtoZm2HkPMlAM9sLe1BZmqek+SJ+/y0muXH7TWWlW1PXbzja21e6vqm5PcUlWnW2sf27lRa+1kkpNJ8ox/emSvfQGwh+2B3JQDzTbBZpp6DDCJEAM9W1uQaa09d6/XqupTVXW8tXZfVR1Pcv8e+7h38fPjVfW+JM9M8pggA8Bq9BRoti0PoIWacfQaYoC+jbVG5sYkL13cf2mSd+7coKqeUFUXL+4fS3JFkjsGayHABjtx5MEuP6neXldjfc1wej7OPf4bBx411hqZ65O8rapenuQTSX4gSarqWUle2Vp7RZKnJXlDVT2SrcB1fWtNkAEY0PJAr6cqzTbT0Nan5wADzMMoQaa19pkk373L86eSvGJx/8+TnBi4aQDsocdpZzuZhrYacwgxqjHQv7EqMgB0ag6BJlGtORdzCDCJEANzIcgAcE56n3a2k2Czv7mEGGA+BBkAzttcqjTLTEPbMrcAoxoD8yHIALAycww0yeZWa+YWYoB5EWQAWLm5TTvbaXuAP+dAM8cQoxoD8yLIALBWc63SJPMMNHMMMIkQA3MkyAAwiDlXaeaynmauIQaYJ0EGgMGp0kzL3AOMagzM01eN3QAANteJIw/OdpB54ujdXQSEHtoIsBsVGQBGZ9rZ8AQYYAxVdTTJ+5NcnK0s8vbW2s9U1ZuS/Mskn11s+rLW2q377UuQAWBSTDsbrh2bYK4VP+jYQ0mubK09WFUXJfmzqvqTxWv/qbX29sPuSJABYJIEmvW9L8BYWmstyfYnDBctbu1c9mWNDACTtr2OZo6frG+voxkiYGxiiJnjvxnowLGqOrV0u3bnBlV1QVXdmuT+JO9urX1g8dLPVdVtVfX6qrr4oDdSkQGgG5uwlmYdVZpNDDHAo+rMw7nw058b6u0eaK09a78NWmsPJ3lGVX1Nkt+vqm9P8pokn0xyJMnJJD+V5HX77UdFBoAuzbVSs8oKTS9XTluHuf27gDlqrf1dkvcmeV5r7b625aEkv5nk2Qf9eUEGgO7NOdCcaxDZ1AADTFtVPXFRiUlVPS7J9yT5v1V1fPFcJXlBko8ctC9TywCYjblOPTubaWcCjGoMTNzxJDdU1QXZKqq8rbX2R1V1S1U9MUkluTXJKw/akSADwCzNMdQcFGiEGGDqWmu3JXnmLs9febb7EmQAmL25Xcp555dsCjCPUo2BzSHIALAx5lylAdg0ggwAG2mOoWbTqcbAZnHVMgA23hyvegYwdyoyALCgStMvQRQ2jyADALsQagCmzdQyADiAT/unTf/AZhJkAOAQDJanx9om2GyCDAAckkHzdOgLQJABALoixACJIAMAZ8UgelyOP7BNkAGAs2QwPTzrYYCdBBkAOAcG1cNxrIHdCDIAcI4MsNfPMQb2IsgAwHkw0F4fxxbYjyADAOfJgHu1rIcBDkOQAYAVMPBeDccROCxBBgBWxCD8/Dh+wNkQZABghQzGz43jBpwtQQYAVsyg/PCshwHOlSADAGtggH4wxwc4H4IMAKyRwfruHBfgfAkyALBmBu1fyfEAVkGQAYABGLxvcRyAVRFkAGAgmzyIt2YIWDVBBgAGtImD+U38OwPrJ8gAwMA2aWC/SX9XYFiCDACMYBMG+JvwdwTGI8gAwEjmOtC3HgYYgiADACOa04BfgAGGdOHYDQCATXfiyIM5/aVLxm7GORFcgLEIMgAwAb2FGQEGOvIPZ9I+9emxW7FyppYBwET0EA5MHwOmQkUGACZkipUZwQWYIhUZAJiYqQQH1RdgygQZAJigMQOEAAP0wNQyAJioIaeZCS5AbwQZAJiw7YCxjkAjvAA9E2QAoAOrCDSCCzAnggwAdORsp5sJL8BcCTIA0Jn9qjOCC7ApBBkA6JTQAmwyl18GAAC6I8gAAADdEWQAAIDuCDIAAEB3BBkAAKA7ggwAANAdQQYAAOiOIAMAAHRHkAEAALojyAAAAN0RZAAAgO4IMgAAQHcEGQAAoDuCDAAA0B1BBgAA6I4gAwAAdEeQAQAAuiPIAAAA3RFkAACA7ggyAABAdwQZAACgO4IMAADQnVGCTFW9uKpur6pHqupZ+2z3vKr6aFXdVVXXDdlGAABgtarqsqp6b1XdscgDr148/1+q6t6qunVxe/5B+7pw/c3d1UeSvCjJG/baoKouSPIrSb4nyT1JPlhVN7bW7himiQAAwIqdSfIfWmsfrqrHJ/lQVb178drrW2v/47A7GiXItNbuTJKq2m+zZye5q7X28cW2b01ydRJBBgAAOtRauy/JfYv7n6+qO5Ncei77GqsicxiXJrl76fE9Sf75bhtW1bVJrl08fOifXPrXH1lz21ifY0keGLsRnDP91y991zf91y9917dvHbsBh/G5Rz5z002ff9Oxgd7uaFWdWnp8srV2crcNq+qbkjwzyQeSXJHkVVX1kiSnslW1+dv93mhtQaaq3pPkybu89NrW2jtX+V6Lg3Ny8b6nWmt7rrth2vRf3/Rfv/Rd3/Rfv/Rd33YM2Certfa8sduwU1VdkuT3kvx4a+1zVfWrSf5rkrb4+T+T/Nv99rG2INNae+557uLeJJctPX7K4jkAAKBTVXVRtkLMm1tr70iS1tqnll7/tSR/dNB+pnz55Q8mubyqnlpVR5Jck+TGkdsEAACco9paJP/rSe5srf3C0vPHlzZ7YbYuDravsS6//MKquifJdyX546q6afH811fVu5KktXYmyauS3JTkziRva63dfojd7zoHj27ov77pv37pu77pv37pu77pv7N3RZIfTnLljkst/3xVna6q25L8qyQ/cdCOqrW25rYCAACs1pSnlgEAAOxKkAEAALrTfZCpqhdX1e1V9UhV7Xn5wqp6XlV9tKruqqrrhmwje6uqr62qd1fVXy1+PmGP7R5emkfpog8jOuhcqqqLq+p3F69/YHGNeCbiEP33sqr69NL59oox2sljVdVvVNX9VbXrAtja8kuLvr2tqr5j6Dayt0P033Oq6rNL595PD91GdldVl1XVe6vqjsWY89W7bOP8G0H3QSZbVzR4UZL377VBVV2Q5FeSfG+Spyf5wap6+jDN4wDXJbm5tXZ5kpsXj3fzhdbaMxa3q4ZrHssOeS69PMnftta+Jcnrk/z3YVvJXs7id+HvLp1vbxy0keznTUn2+y6I701y+eJ2bZJfHaBNHN6bsn//JcmfLp17rxugTRzOmWx9OePTk3xnkn+/y+9O598Iug8yrbU7W2sfPWCzZye5q7X28dbal5K8NcnV628dh3B1khsW929I8oIR28LBDnMuLffp25N89+JSi4zP78KOtdben+Rv9tnk6iT/p235iyRfs+NypozoEP3HRLXW7mutfXhx//PZuprupTs2c/6NoPsgc0iXJrl76fE9eew/QMbxpNbafYv7n0zypD22O1pVp6rqL6pK2BnPYc6lL2+zuIz6Z5N83SCt4yCH/V34fYupEW+vqst2eZ1p8n9d/76rqv6yqv6kqr5t7MbwWIvp0s9M8oEdLzn/RnDh2A04jKp6T5In7/LSa1tr7xy6PZyd/fpv+UFrrVXVXtcD/8bW2r1V9c1Jbqmq0621j626rUD+MMlbWmsPVdWPZqu6duXIbYJN8OFs/V/34OI7Nf4gW9OUmIiquiRb30b/4621z43dHjoJMq21557nLu5Nsvyp4lMWzzGA/fqvqj5VVcdba/ctSrD377GPexc/P15V78vWpyGCzPAOcy5tb3NPVV2Y5B8n+cwwzeMAB/Zfa225r96Y5OcHaBer4f+6ji0PjFtr76qq/1VVx1prD4zZLrZU1UXZCjFvbq29Y5dNnH8j2JSpZR9McnlVPbWqjiS5JokrX03DjUleurj/0iSPqbBV1ROq6uLF/WPZ+kbYOwZrIcsOcy4t9+n3J7ml+ebdqTiw/3bM6b4qW3PB6cONSV6yuHrSdyb57NLUXSauqp68vZ6wqp6drTGaD4EmYNEvv57kztbaL+yxmfNvBF1UZPZTVS9M8stJnpjkj6vq1tbav66qr0/yxtba81trZ6rqVUluSnJBkt9ord0+YrN51PVJ3lZVL0/yiSQ/kCS1dSntV7bWXpHkaUneUFWPZOsX+/WtNUFmBHudS1X1uiSnWms3ZuuX/W9V1V3ZWth6zXgtZtkh++/HquqqbF2l52+SvGy0BvMVquotSZ6T5FhV3ZPkZ5JclCSttf+d5F1Jnp/kriT/L8mPjNNSdnOI/vv+JP+uqs4k+UKSa3wINBlXJPnhJKer6tbFc/85yTckzr8xlXMEAADozaZMLQMAAGZEkAEAALojyAAAAN0RZAAAgO4IMgAAQHcEGQAAoDuCDAAA0B1BBmCDVdU/q6rbqupoVX11Vd1eVd8+drsA4CC+EBNgw1XVzyY5muRxSe5prf23kZsEAAcSZAA2XFUdSfLBJF9M8i9aaw+P3CQAOJCpZQB8XZJLkjw+W5UZAJg8FRmADVdVNyZ5a5KnJjneWnvVyE0CgANdOHYDABhPVb0kyT+01n6nqi5I8udVdWVr7Zax2wYA+1GRAQAAumONDAAA0B1BBgAA6I4gAwAAdEeQAQAAuiPIAAAA3RFkAACA7ggyAABAd/4/9iY7FKsYA7wAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "plt.rcParams[\"figure.figsize\"] = (10,6)\n",
        "plt.plot(x, y_f1, label = 'F1-score')\n",
        "plt.plot(x, loss, label = 'Loss')\n",
        "plt.xlabel('x')\n",
        "plt.ylabel('Score')\n",
        "plt.title('Distilbert on SQuAD dataset, 1-D interpolation (0 - Initial parameters, 1 - SQuAD minimizer)')\n",
        "plt.grid()\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "T6WYW30exLET"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "lista=pd.read_csv(\"./drive/My Drive/zdata_10.txt\", sep=\",\", header=None)"
      ],
      "metadata": {
        "id": "wLhlNYfMkj13"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "z = lista.iloc[:, 0].to_numpy()\n",
        "z1 = np.split(z, 10)\n",
        "z1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9zOdq_CylV8u",
        "outputId": "76167acb-895e-4a1a-9b75-1237dbd1fc23"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([0.11890606, 0.        , 0.        , 0.        , 0.        ,\n",
              "        0.11890606, 2.14030916, 1.78359096, 0.83234245, 0.59453032]),\n",
              " array([0.11890606, 0.        , 0.        , 0.        , 0.        ,\n",
              "        1.30796671, 9.15576694, 4.39952438, 0.59453032, 0.71343639]),\n",
              " array([ 0.        ,  0.        ,  0.        ,  0.        ,  1.07015458,\n",
              "        18.43043995, 19.85731272,  6.3020214 ,  2.25921522,  0.35671819]),\n",
              " array([ 0.        ,  0.        ,  0.        ,  0.47562426, 11.77170036,\n",
              "        41.14149822, 27.11058264, 10.10701546,  2.02140309,  1.07015458]),\n",
              " array([ 0.        ,  0.        ,  0.47562426, 12.960761  , 54.69678954,\n",
              "        49.8216409 , 30.32104637,  9.98810939,  2.37812128,  0.95124851]),\n",
              " array([ 0.        ,  2.85374554, 18.66825208, 62.42568371, 67.65755054,\n",
              "        53.62663496, 29.13198573,  8.20451843,  1.90249703,  0.35671819]),\n",
              " array([ 4.6373365 , 27.94292509, 61.59334126, 70.51129608, 66.34958383,\n",
              "        49.94054697, 20.09512485,  4.04280618,  0.83234245,  0.83234245]),\n",
              " array([14.74435196, 49.70273484, 63.49583829, 63.73365042, 52.91319857,\n",
              "        24.97027348,  5.11296076,  1.18906064,  0.47562426,  0.23781213]),\n",
              " array([21.16527943, 43.63852556, 43.99524376, 31.39120095, 12.60404281,\n",
              "         2.73483948,  0.35671819,  0.11890606,  0.35671819,  0.23781213]),\n",
              " array([ 7.37217598, 10.34482759,  6.89655172,  2.25921522,  0.47562426,\n",
              "         0.47562426,  0.47562426,  0.59453032,  0.35671819,  0.11890606])]"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "z_loss = [] \n",
        "for i in range(0,10):\n",
        "  z_loss.append(\n",
        "      [100 - z1[i][j] for j in range(0,10) ])"
      ],
      "metadata": {
        "id": "KwWzfsekl8Ko"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Trajectory projection"
      ],
      "metadata": {
        "id": "3gdWebgmM0Tp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def run_trajectory_projection(args, model_path_0, model_path_1, checkpoints_list):\n",
        "\n",
        "  model_0 = AutoModelForQuestionAnswering.from_pretrained(model_path_0) \n",
        "  model_1 = AutoModelForQuestionAnswering.from_pretrained(model_path_1) \n",
        "\n",
        "  sd_0 = model_0.state_dict()\n",
        "  sd_1 = model_1.state_dict()\n",
        "\n",
        "  delta_1 = state_dict_diff(sd_0, sd_1)\n",
        "\n",
        "  x = []\n",
        "  y = []\n",
        "\n",
        "  sqlen_1 = sum([torch.sum(torch.square(t)) for t in delta_1.values()])\n",
        "\n",
        "  for i in tqdm(range(0, len(checkpoints_list))):\n",
        "\n",
        "    model_n = AutoModelForQuestionAnswering.from_pretrained(checkpoints_list[i])\n",
        "    sd_n = model_n.state_dict()\n",
        "    delta_n = state_dict_diff(sd_0, sd_n)\n",
        "\n",
        "    sqlen_n = sum([torch.sum(torch.square(t)) for t in delta_n.values()])\n",
        "\n",
        "    v_cos = state_dict_mult_scalar(delta_n, delta_1) / np.sqrt(sqlen_n * sqlen_1)\n",
        "\n",
        "    d_alpha = v_cos * np.sqrt(sqlen_n / sqlen_1)\n",
        "\n",
        "    d_beta = np.sqrt(sqlen_n / sqlen_1 - d_alpha * d_alpha)\n",
        "\n",
        "    x.append(d_alpha)\n",
        "    y.append(d_beta)\n",
        "    \n",
        "  return x, y"
      ],
      "metadata": {
        "id": "aX6qyqd5M5kq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_path_0 = \"./drive/My Drive/distilbert-init/checkpoint-init\"\n",
        "model_path_1 = \"./drive/My Drive/distilbert-squad-100/checkpoint-48000\"\n",
        "checkpoints_list = [\n",
        "                    \"./drive/My Drive/distilbert-squad-100/checkpoint-8000\",\n",
        "                    \"./drive/My Drive/distilbert-squad-100/checkpoint-16000\",\n",
        "                    \"./drive/My Drive/distilbert-squad-100/checkpoint-24000\",\n",
        "                    \"./drive/My Drive/distilbert-squad-100/checkpoint-32000\",\n",
        "                    \"./drive/My Drive/distilbert-squad-100/checkpoint-40000\",\n",
        "                    \"./drive/My Drive/distilbert-squad-100/checkpoint-48000\",\n",
        "                    ]\n",
        "\n",
        "x_cp, y_cp = run_trajectory_projection(args, model_path_0, model_path_1, checkpoints_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HKcxuz26mUa7",
        "outputId": "b012475e-e3a7-4e5f-cd61-1b1b308a756c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  0%|          | 0/6 [00:00<?, ?it/s]\n",
            "100%|██████████| 102/102 [00:00<00:00, 1406.93it/s]\n",
            " 17%|█▋        | 1/6 [00:03<00:17,  3.44s/it]\n",
            "100%|██████████| 102/102 [00:00<00:00, 1144.78it/s]\n",
            " 33%|███▎      | 2/6 [00:06<00:13,  3.28s/it]\n",
            "100%|██████████| 102/102 [00:00<00:00, 1456.06it/s]\n",
            " 50%|█████     | 3/6 [00:09<00:09,  3.11s/it]\n",
            "100%|██████████| 102/102 [00:00<00:00, 1875.48it/s]\n",
            " 67%|██████▋   | 4/6 [00:11<00:05,  2.82s/it]\n",
            "100%|██████████| 102/102 [00:00<00:00, 1396.51it/s]\n",
            " 83%|████████▎ | 5/6 [00:15<00:02,  2.96s/it]\n",
            "100%|██████████| 102/102 [00:00<00:00, 1691.65it/s]\n",
            "100%|██████████| 6/6 [00:17<00:00,  2.86s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_cpa = [0] + [it.item() for it in x_cp]\n",
        "y_cpa = [0] + [it.item() for it in y_cp]"
      ],
      "metadata": {
        "id": "pzqb19m-xrfA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_cpa"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "flUME6193sW6",
        "outputId": "3befb4e4-0a8a-4ac9-8a79-175552d45f69"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0,\n",
              " 0.4369814395904541,\n",
              " 0.6985232830047607,\n",
              " 0.8576564788818359,\n",
              " 0.9463721513748169,\n",
              " 0.989151120185852,\n",
              " 0.9999998807907104]"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "alpha_range_x = [-1, 2]\n",
        "alpha_range_y = [-1, 2]\n",
        "n = 10\n",
        "\n",
        "x_ip = [alpha_range_x[0] + (alpha_range_x[1] - alpha_range_x[0]) * i / (n - 1.0) for i in range(0, n)]\n",
        "y_ip = [alpha_range_y[0] + (alpha_range_y[1] - alpha_range_y[0]) * i / (n - 1.0) for i in range(0, n)]"
      ],
      "metadata": {
        "id": "VwKiKV5fmV-6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.rcParams[\"figure.figsize\"] = (15,10)\n",
        "fig,ax=plt.subplots(1,1)\n",
        "\n",
        "X, Y = np.meshgrid(x_ip, y_ip) \n",
        "\n",
        "cp = ax.contourf(X, Y, z_loss, levels = 20)\n",
        "fig.colorbar(cp) # Add a colorbar to a plot\n",
        "#plt.plot(y_cpa, x_cpa)\n",
        "#ax.scatter(y_cpa, x_cpa, label = 'SQuAD optimization trajectory')\n",
        "ax.set_title('Distilbert on SQuAD dataset, 2-D interpolation between initial parameters and SQuAD minimizer')\n",
        "ax.set_xlabel('x')\n",
        "ax.set_ylabel('y')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 638
        },
        "id": "SAOuRovUwvyq",
        "outputId": "0525fdac-c0d4-4c60-ccd2-419e86de799f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No handles with labels found to put in legend.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1080x720 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAAJcCAYAAAA1ngF6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdebwkd1nv8c+TmTMTwpkkEwYHSAjBhB0BISIuIBh5XUCu4bogKBrcYhARvK6IV0AREFcUrxgFWWURBEFR4KqIiIkGDGswJASykcAkM5lzCJkl89w/qmpOn56u7qrqX1X9flXf9+t1XjOnu093nT691Lef5/eUuTsiIiIiIiJDcFzfGyAiIiIiIhKKAo6IiIiIiAyGAo6IiIiIiAyGAo6IiIiIiAyGAo6IiIiIiAyGAo6IiIiIiAyGAk4izOyVZvZ/Gvzc6Wa2bmZb8u8/YGY/nv//6Wb2odDbKvWY2efN7Dv63o7YmdmvmNmf970dVZjZC8zsDUv8/KfM7NEBN6m43qPP/yGqc78tuqyZ/b2ZnVfxuvQcjoiZnWFmbmZb+96WvpjZI83sv0NfdsbPbtrHEImFAk4E8jfHr5rZmpntM7MPm9kFZnb07+PuF7j7b1S8rqNvtO5+tbuvuvvtbW3/xG13GpjM7Fwzu9TM9pvZHjP7JzO758T59zezd5nZLfl9+09m9ogGt/P0/M3y+6dOf7SZHclf3NfN7Foze6uZfUOI369kW9zMzmrr+pvejpn9gpl9Mr+frzKzX1hw+deY2cH88mv5z77EzE4q+xl3f7G7V9o5XzZgdCm/L140eZq7P8DdP9DTJs2UQjiqc79NXnbW48XdH+/urw2/lekYanAzs2/N32dvMbObzezfJl+3zew0M3ujmd1kZl8xs/8wsyc0uJ1H56+lvzR1ehHAiveOG83sb83ssSF+PwB3/1d3v0/oy8742c72MUTqUMCJx/909x3APYCXAr8EvKrfTaqu60/K8p3v1wE/B5wE3BP4Y+D2/PwzgX8DPpGfdzfgncD7zezhNW/uPOBm4IdnnHe9u68CO4BHAJ8B/tXMzqn7OyXOyO6fncDjgJ82s6cs+JmX5Y/5OwM/Qnb//ZuZ3bHVLa1gzJ/8SrxSelxaJrp9DDM7Efhb4I+AU4BTgRcCB/LzTwE+BBwEHgDsAn4feLOZPanmzc177wA4OX//eDDwfuAdZvb0mrcxOCk9ziVi7q6vnr+AzwPfMXXaw4EjwAPz718DvCj//y6yF+h9ZC+e/0oWVl+f/8xXgXXgF4EzAAe25j/7AeDH8/8/nSwEvAK4hWzn/JyJbTiJLGR9EbgOeBGwZepnfx+4CXg7cBtZwFgH9pX8rncD3pVv9xXAT0yc9wLgrWTBZQ34FHB2yfV8L3DpnPv09cB7Zpz+J8A/5/9/NHDtvL8FWeA8AnwPcBi4y8R5x/x8fvorgEvmbNsPAV/I77fnTd5m/nf/9/xv+8X8urbl530w/1t+Jb+Pv58sUPwt8GVgb/7/0yZu6+nA5/L78yrgByfO+1Hgsvzn3gvco+x2Gjym/xD4oznnv4b88Txx2o78d/7pkp95AfCG/P9n5Nt4HnA1sAd4Xn7e48h2Tg7l2/+xBo/nF7H4+bHosfyGie//Crghv54PAg/ITz8/386D+ba+e/pxCGwH/gC4Pv/6A2D75GOQLOh/Kf/dfmTO/f4B4CXAfwD7gb8BTpk4/xHAh8kefx8DHp2f/ptkz+3b8u18BdlO4R/l56/kj5ffzr+/Q37ZU+Zdb8W/y4eA3yF7nF4FPL7KaykLXk+Ky1L+ePkAG6+VZwL/RPbY2AO8kWzntPQ1fOqx/kqyHdg14F/In2v5+S8Hrsn/Hh8BHjn1OHob8Ib8/B9nzmtE/jMO/BTw2fz2fiPf/g/n1/HWqcs/Ebg0v74PAw+aeA3d9H5S4W/5AbLHyr/lP3cWc16Dpu6nKr/XBfnvtY/sAy3Lz9uSP0b25Lf1TCbe96Zu52xK3p/y838D+CRw3NTpv5RftzH1vjr9eMm/v2P+Oz+F7PE1+dg75ufz038euHH6tpv8bZl6fyJ7jP488HGy16G3AMfPuewv5Jf9Ctnzczfw9/nt/j9g5/TvAnxT/lgpvm4DPp9f7jjgl4EryZ5Hb2Xj9aG4jh8jez3/YNX3Gn3pq+yr9w3QV/mbY/5Ef0b+/9ewEXBeQvaGuZJ/PXLihX7TdU2/kHJswDkM/Gx+Pd+fv/AVLzrvAP40f6H+GrKdop+c+tln5S9sd8hP+9CC3/WDwP8FjgceQrZj/u35eS/IXxCfQPaG9RLgopLr+dr8sr8PPAZYnTr/Bmbs6OWXPZzf/qYX9ZL77/8A/5H//xPAz02cd8zP56d/O9mOwR1nnHf//IX/UWQ7rr+Xb0+xU/Ywsh2Irfnf7jLgORM/78BZE9/fiSx8nUAWEP4KeGd+3h3J3vTuk39/VzZ2rM8l2ym/X35bvwp8uOx2aj6eDfgv4II5l3kNUwEnP/11wFtKfuYFHBtw/ix/7D2Y7BPY+01fduLnmzye5z0/Fj2WJwPOj+Z/nyKsXDrvvmDzjvqvAxfl23xnsp2Z35h4DB7OL7NC9ty5lXznY8Z9+AGyEPHA/H54+8R9eirZjscTyHZGHpt/f+fp146Jx/kn8v9/M9mOy8UT532s4vUu+rscAn6C7DXhGWQhzxa9lrLg9WTGZacfL0d/X7Id9cfmf78753/7Pyh73ZjxWF9j4zn/ciZeJ4GnkT2Pt5IF1RvY2PF8Qf77Pym/7+5AtdeIvwFOJKtAHAD+kew18yTg08B5+WW/niwYf2N+H52X/y7bZ/1eFf6WHyB733pAvn0nUfIaNON+qvJ7/S1wMnA62fPtcfl5F5B9AHF3sqrMP1MecE7Mt/m1wOOZeq6QPddeOOPn7plf572oFnB+iCyobQHezcQHPrN+Pj/9a/PT71dyH9X52z6aY0PLf5B9MHNKfv9eMOeyF5GFmlPzx8hH88fL8WRh//kLfpcVsjD/kvz7Z+fXeRrZ8+BPgTdNXcfryF4H7lD23qEvfVX9iq58LJtcT/ZCNO0Q2RvFPdz9kGf9s97wNr5E9kZ9yN3fAvw38J1mtpvsTew57v4Vd/8SWZiYbDu63t3/yN0Pu/tXF92Qmd0d+Bbgl9z9Nne/FPhzNpfvP+Tu7/Gsn/f1ZDuux3D3z5G9KJ9K9knQnnwtw2p+kV1kby7TijecWffrLD8M/GX+/7+kvNVg0vVkO/knzzjve4G/dfcPuvsBsgB1pDjT3T/i7hfl9+nnyd4Evq3shtz9Jnd/u7vf6u5rZJ+eTl7+CPBAM7uDu3/R3T+Vn34B2RvPZe5+GHgx8BAzu0eF32+RF5Dt/PxFg58te8yXeaG7f9XdP0b2afLMx8sSj+ey50eVx/JR7v5qd1/L/+YvAB48b73RlB8Eft3dv+TuXyarnPzQxPmH8vMPuft7yAL0vH7617v7J939K2SPvyfnC4SfRlb1fI+7H3H39wOX5PfbLP8O3MvM7kS28/4q4NT8OfhtZDs3zLvein+XL7j7n+WvCa8le+3bvfBey1R6PVnE3a9w9/e7+4H8b/B7zHlezvB3E8/55wHflD+GcPc35M/jw+7+u2Q7f5N/v39393fm991XK75GvMzd9+fP908C73P3z7n7LWSfwn99frnzgT9194vd/XbP1hwdIAsas1R5jLzG3T+Vv64cpvw1aJOKv9dL3X2fu19NFmIekp/+ZLLn6TXufjNZmJ3J3fcD38rGByRfztdqFo+pee8dkAXcKs4j+7DmdrL3jqeY2cqCn7k+/3fea2DVv+0sf+ju1+f30bvZuP9m+SN3v9HdryPrErnY3f/L3W8j+1Bi3u1AVsVfI3u8Q/ae8zx3v3bidfB7p9rRXpC/DizcnxBZRAEnbqeStb9M+22yT9/fZ2afM7NfXuI2rpsKR18g+4TnHmSfwHwxH3ywj+wN52smLntNzdu6G3BzviM+eXunTnx/w8T/bwWOL+vHzd8Mn+zudyarYj2KjRfTPWQ7QtPuSvbGdtOijTWzbyH71O7N+Ul/CXydmc17U4Ds93GyNoppd2Pifst3Mo9ui5ndO19seoOZ7ScLHrvmbOMJZvanZvaF/PIfBE42sy35dX8/2RvLF83s78zsvvmP3gN4+cTf9mayUHbqrNupysx+mmwn/zvzN7Fi+lmxmPaVC66i7DFfZvrxslpyuaaP57LnR5XHMgBmtsXMXmpmV+Z/o8/nZ5X+XafcLb/u6W0o3JTvTBbm3Q+w+ff8Atn9sovsPvq+4v7J76NvZfbziHwn5BKyndBHkQWaD5MFv8mAM+96q/xdjv6N3f3W/L/zfr9JlV9P5jGz3Wb2ZjO7Lv8bvoHqfz/Y/JxfJ3uM3y2/7p83s8vyBe/7yD6J3zXrZ/PLV3mNuHHi/1+d8X1x/90D+Lmpv83d2fz4mlTlMTL9+lb2GrRJxd+r7Pm+6XWVzc+XY+Qf7Dzd3U8jq2bejayyCvPfO4rz58rD62PIWhkhq7ocD3zngh8tXj/mvQZW/dvOUvX1cqnbMbOfJPsA8gfcvfgA7x5ka4yKx81lZG2vkx9W1N2nECmlgBOpfKLLqWT955vknwT/nLt/LfBdwP+eWNRet5JzqpnZxPenk32KdA3ZJ3m73P3k/OtEd3/A5KZMb9qC27oeOMXMdkzd3nU1t/kY7v6fwF+TvVlB1iP8fTMu+mSyNpUDZL3FJxRn5J9iT346dx7ZTv+lZnYDcPHE6fP8L+Cj+Zv7tC+S7UAUt3kCWXtK4U/IWi3u5e4nAr+Sb0OZnyP7tPcb88s/qrhqAHd/r7s/luzN+TNkn1hC9vf9yYm/7cnufgd3//CC362Umf0oWY/1Oe5+bXG6Z9PPVvOvC+b8/CrZmoh/bboNE6Yfi00ez1D+/KjzWP4BspbA7yDbeT0jP7243irPm8nKWrENTd194v+nk1WA9pDdR6+fekzc0d1fOmc7/4WsHe3rgf/Mv/8fZOspPphfZt71Vvm7dGHR3+DF+WW+Ln+ePY35z8tpk8/5VbJP6K83s0eSrZV8Mlmr1MlkbZCT1z29bXVfI+a5BvjNqb/NCe7+ppLbXvQYOeZn5rwGTVvm99r0ukr2uK7E3T9D1kY4+d7x3TMGJDyZbL3bFWTvHTDx/gHcZeL/P0S2f/Xu/L3jc2QBp8p7x5fIKsVJyh/TvwGcm1fLCteQrZ+bfOwcn1eICk07UUSOoYATGTM70cyeSFY1eIO7f2LGZZ5oZmflO163kH0KUnxKciNZP25VXwP8jJmtmNn3ka3JeI+7fxF4H/C7+TYdZ2Znmtm8towbgdPMbNusM939GrJPeF9iZseb2YPIFhXWHudr2ZjPnzCzr8m/vy9Z2Lsov8gLgW82s980s1PMbIeZPYtsWtev5Ze5nOwT3e/MWwd+law9BDM7nuwN7XyyMn7x9SzgB6Y/BbbMqWb2fLKFwL9SsulvA56Yb/82srUTk8/DHWQ96+v57/SMqZ+f/vvuIPs0bZ9l03+eP7FNuy0bpX1Hsp3IdTYeJ68EnmtmD8gve1L+9y+7nWJ09KNn/VJm9oNkO4GP9ax9sDIz225mDyObcreXZq1t024Ezih2Uho+nqH8+VHnsbyD7P6/iWyH6MUztnXec/ZNwK+a2Z3NbBfZ43eZEdhPs2yE+glkj7+35W00bwD+p5n9j7zqdLxlY25Pm7Od/0JWsfu0ux8kX4cAXOVZKxfzrneJv0tomx4vM+wge/7cYmanki3AruMJE8/53yD7kOWa/HoPk60n2Wpmv0a2vmKeRa8RdfwZcIGZfWP+GnbH/PWwCO7Tf/NFj5FNFrwGhfy93kr2PD3NzHaSfdAyk5nd18x+rtjmvNryVDbeO36ffPCFmd0l/x2fStbO+XzPWvO+TPZhxtPy++FHyRb7F84jew+afO/4HrLHweQHWsU27bas+v184LkTVY+k5PflW4EfdvfLp85+JfCblrdB569n53a9jTIeCjjxeLeZrZF9yvE8sh7vHym57L3IPmVaJ+uD/7/u/s/5eS8h2xnaZ2Y/X+F2L86vbw/Z+o3vdfeiZeqHgW1kCxf3ku2cz2xXyf0T2aSiG8ysrIz/VLJPsK8n6+N9vrv/vwrbOW0fWaD5hJmtA/+QX9/LANz9s2StEw8mawnaR7Zj8b+K28t7ln+KbO3EdWSfyhWVhyeRBYfXufsNxRfwarJFsI/LL3e3/PbXyT7B/jqyqULvm7XRed/0M8na3b5Idr9eO3GRnyf7xH+NbOfjLVNX8QLgtfnf98lkbRV3IPv7XZTfD4XjgP9Ndl/fTNY29Ix8O94B/BbZ6NP9ZL3cjy+7nfyNa41s0MIsLyKrRP2nVW9H+8X8MX8T2eLSjwDfXFL5quuv8n9vMrOP5v+v+3iG+c+Pqo/l15G1zFyX3/ZFU+e/Crh/fl+/c8bPv4isFezjZPf/R/PTmno92SfWN5B9qvwzcPQDiHPJwvmXyV6LfoGN94mXk/XM7zWzP8xP+zDZ46+o1nyabGF/8X2V623ydwlt1uNl0guBh5J9oPR3ZNXiOv6SbOf1ZrLF9E/LT38v2XP2crLHyG0sbtNZ9BpRmbtfQjbA4RVk9/0VZIMdCpveTyr8LaeVvgYF/r3+jOy+/BjZ82Pe32eNbKjCxWb2FbLn4yfJquHkz+9vJXtufJrstf11wDPd/dUT1/MTZL/7TWQL/j8MYNmx1u4B/PHke4e7v4vs/n3qxHXsy7fhE2TrmL5v6jZScw5Zy9nbJt4HijVXLyebOvm+/HX/IrK/g0grislbIoOWf1p3EdlOaDLHF4qFmT2NbPrRc/velq5YdjyKH3f3b+17WyRdZvYasglVv9r3tkh9lh0359+Ad7j7ry26vIjEQRUcGYV8TcjjgbvaxqQ1qcizSU+jCTciInB06toTgNvN7C6LLi8icegt4JjZ3c3sn83s02b2KTN79ozLmJn9oZldYWYfN7OH9rGtMgzu/gl3f5FnU4xEREQW8mz89AvzNmURWZKZvdrMvmRmn5w47RQze7+ZfTb/d2d+eqMs0FuLmpndFbiru380X9D4EeBJ7v7pics8gWxR9xPIejVf7u7q2RQRERERSZCZPYp8fZu7PzA/7WVkh194qWWHP9np7r/UNAv0VsHx7IBfH83/v0Y2E336GBLnkv3y7u4XkR3fo+vFpyIiIiIiEoC7f5Bjj/d0LtnBnMn/fdLE6bWzQO0DnrXBzM4gO47CxVNnncrmiTLX5qdtOsqwmZ1PNs6XO5xgD7vHmYsOFlzuNm/+s9KP4+1Q35sQpe2mASIiIiJt+tjHD+3x7IDjUTvnMcf7TTd3M4H8Yx8/9CmyqZCFC939wgU/tjs/dABkUz6Lg8BWygLTeg84+YLvtwPPmTooVGX5nXYhwP0etN1f8+6ygzBvdvnB3YsvJINy7203Lr7QCJy1olAoIiKyrF2nXveFvrehiptuPsI//v3XdHJbu0697jZ3P7vpz7u7my33KW2vAceygyu+HXiju8+aW38dm49OfBoNj3qvMCPQ7HEwxFB0xaHySqXCj4iIiHTsRjO7q7t/MW9B+1J+eqMs0FvAMTMjO8DdZe7+eyUXexfw02b2ZrKFRbdMlK9mus1XFGYkqHmPp7GFH1AAEhERkeDeBZwHvDT/928mTq+VBaDfCs63AD9EdiT6S/PTfgU4HcDdXwm8h2xqwhXArcCP9LCdg3LlgWbh78ztw9uRD2FRmB5bAFL4ERERkXnM7E3Ao4FdZnYt8HyyYPNWM/sx4AvAk/OLN8oCvQUcd/8QYAsu48Azu9mi4WkaZkJe19iDkao/GQUfERGJ3e23n8Te9Wdz+PYz8IldVMPZuuXz7Fx9OVu23NLjFg6Duz+15KxzZly2URbofciAhBMy0ISiYFRuTOFnVvBR6BERkZjsXX82J+98KKfs3Eq2kiLj7ty89xT27n02u0769R63UKpSwElYjIEmlLEHozG0vin0iIhITA7ffsYx4QbAzDhl51b27Dmjnw2T2hRwEjHkMBPSWILRUKs/anETEZG+OHZMuCmY2aa2NYmbAk6kFGi6VfX+TiEIDbH6o2qPiIiIVKWAEwkFmjSU/Z1SCD6FoVR/FHpERERkFgWcnvQdaK66dVely93zhD0tb8kwzPt7Kvx0Ry1uIiLSlOG4+8w2NXfH8B62SppQwOlA32GmUDXULPszkxSQhlH1gbRb31TtERGRRbZu+Tw37z2lZIraYbZu+Xx/Gye1KOC0IJZAA8sHlL5vf8gBaSjBpzAZgGIOOwWFHhERmbRz9eXs3fts9uwpPw6OpEEBJ4CYAk2h72ATyhgD0hCCT2php6AWNxGR8dqy5RYd52YgFHBqijHMFIYSakIKcZ/EEpJSXeeTatiZpGqPiIhIOhRwFog50IBCTRdm3cexhJ5CKlWfIYSdgkKPiIhInBRwpsQeaAoKNv0qu/8VfKobUtgpqMVNRESkf6MOOKmEmULoUHP1+snHnHb66r6gtzE2KVR7IL7gU4SdoQSdadPBR4FHRESkPaMKOKkFGminUjMr2FQ5rwkFpnSqPdD/Op8hVnVmKQKPgo6IiEh4gw44KQYaaK/9LHR46fo2hxaWUqn2FLqu+owh7FxxaEUhR0REJLDBBZwDR7Yq2EzpI9i0YQzVpZSqPYXJ55vCTn2q5oiIiIQ1uICTmjaHBQwl2LSl7v3TZyBKpdpThJ0229mGGnZUzREREQlDAacHbU9AU7BpR2xDGWIOPV0EHRhe2FE1R0REZHkKOB1SsBmeFEIP9Bd8ugo6MKywo2qOiIhIcwo4LevieDUKNnGJLfRA/9WeLoMODCPsqJojIiLSjAJOC7o6CGfTYLNnfTXwlpTbtbre2W3FbPpv1XfggX6qPV0HHUg/7KiaIyIiUo8CTiBdhRpYrmLTZbjp4vZSDVAxVnkKXVR7+gg6kG7YUTVHRESkOgWcJSnY9GtIASq10APLB5++gg6kGXZUzREREVlMAaeBLkMNKNj0adb9p9CzYfq50DTw9Bl0YCPspBB0VM0RERGZTwGnoq5DDSjYxGr6vu26TS7m0FM8T1IPOhB/2FE1R0REZDYFnAUUbGSRvqs8EF/oST3oQBphR9UcERGp4oDb0feMMVDAmaGPUAMKNkPSd5UH4pjcNoSgA/GHHVVzRERENijg5PoKNbD8cWyahJv1teOXus0qVnfc1vptpCLGwAPdhZ6hBB2IN+yomiMiIpIZfcBRsGlPl7c1S8wBK4a2Nug+9Fx1666lJq/FFHQgzrCjao6IiIzdKANOn6EGhh9sYhH6d247MMVQ5YH2Q8+y1RyIL+hAXGFH1RwRERmz0QScvkMNKNikbvq+HEvggXZCz1CDDsQTdlTNERGRMRp0wIkh1EA/wQYUbto2ef920Q4XS1tbIdQQgyEHHeg/7KiaIyIiYzO4gHPwyFYFGwWbznVd3SnEWOVR0CnX5wFFVc0REZGxGFzAicGywQbUjpa6MQceBZ3F+go6quaIiMgYKOAElFKw8bX2D/ZkO7QTVYgl8EB3oUdBZ7HLD+5WNUdERCQwBZwAFGz6v63Q2g5nfQUe6L7Ko6Azn6o5IiIiYSngLCGlYANpB46uld1XbQWfMQQeBZ35VM0REREJQwGngb6CDcRftRm66ftSgac+BZ1yquaIiIgsTwGnhhDBBobZjjZWYww8ECb0KOiUUzVHRESkOQWcChRspKrJ+77NdTx9Bh4IW+UJEXSWCTkQZ9BRNUdERKQZBZwFUlpno2ATl66qOxBX4Gkadq5eP7nXag7EG3RUzREREalOAadESsEGmoWbrfu3NLqtKg6feHtr152qrqo7EEdLW5OgE0PbGsQXdFTNERERqU4BZ4qCTRhd3EaXQge2Lqs70E/g2bO+ulQ1BxR0pqmaIyIispgCTm4Mk9GGFjq6NH3fDSnwtBl2lqnmgILOLKrmiIiIzDf6gDOGAQIKNuF1GXiGUN1R0AlP1RwREZHZRhtwFGwkpMn7ekjVHQgbeBR0wlI1R0RE5FijCzhjCDagcNOnIbezQZjAo6ATVp/VHFDQERGRuIwm4KQWbEBVm6EYUjsbhA08Cjrh9FXNAbWtiYhIXAYfcPoMNhB3O9rK2sb/D+2o/ePS0JDa2SDMwAIFnXBUzRERkbEbbMBRsCk3GWzmnSbHCh0Eh1rdUdDpN+iomiMiImM2uIBz4MjW5NrRumxFU5BZzvT912bgSbm6M4Sgs2zIgTiCjqo5IiIyNoMLOCHEHGwgXNVGltdmm98QhhWkHHRCVXOg36Cjao6IiIzNcX1vQEz2rK/WDjfra8fXDje+ttK4alM33KysKdx0pbiv27rPi79/k8dBFcXjsmnwnqfJ82RSk+fmpGWqulfduuto2FnWlQd2Hw07XSuCTteuOLRytKIjIiICYGbPNrNPmtmnzOw5+Wm/bWafMbOPm9k7zKzxm7cCDt0FG2jejqZgk54uA09ok2EnZOAJEXSaunr95NEHncsP7u416IiIiJjZA4GfAB4OPBh4opmdBbwfeKC7Pwi4HHhu09sYdcBp+qlw02BTd0dRwWZYhlLdCRF4lgk6Iao5MQWdPqiaIyIiPbofcLG73+ruh4F/Ab7b3d+Xfw9wEXBa0xvQGpyaYh4ioGDTzLYK99vBwOtrUh5WAOGmsy2zRmcy5DRZozP2iWuTIafr9TkaQiAi0q3bfKXDD7c+v8vMLpk44UJ3v3Di+08Cv2lmdwK+CjwBmLw8wI8Cb2m6BQo4kpwqgaTN2wwddmA4wwqahp0+hxFMVnOWGUYAy4WdyWpOX2FHQUdERALY4+5nl53p7peZ2W8B7wO+AlwKHN05MbPnAYeBNzbdAAUcaVUfYaRtKYcd6G4UdZOws+xBQ0NVdWCcYaevqs5k25rCjojI8Ln7q4BXAZjZi4Fr8/8/HXgicI67e9PrV8CRo4YYRto2fZ+12cqWathZtqoDamErKOyIiMgQmNnXuPuXzOx04LuBR5jZ44BfBL7N3W9d5voVcEZMgSa8Nqs7qYYdtbCFqerA+NbrqIVNRGSw3p6vwTkEPNPd95nZK4DtwPvNDOAid7+gyZUr4Ii0pKuwA19LvhMAACAASURBVOkMKVALW9pVHehnvY4OGCoiMizu/sgZp50V6vpHHXB2ra4vNW42JpqgFreU1+20HXb6bmFrEnRguRa2Nqo6MOwWNlVzRESkqlEHnDFTe1p/FHY267uFbShVHRhH2FHQERGRRRRwRHo0lLADYQLPmFvYQlZ1YPhhR0FHRETKKODUtLrjtsZHYK+qjSPRS/w0kW0ztbCFCTrQ/3CCtoOOQo6IiEwaXMDZftzhvjdBJIghTGSLoaoD6bawtRV0YFhVHVVzRERk0uACjsgQpTqRra2qDqTdwtZ3+xoMM+wo6IiICCjgDGqSWlUaMJC2VNftxNrC1uexdWJoX4PhhR0FHRGRcRtkwDl9dd+mtpCh04jodmzb7zNPP3iidbwl5RR20q7qxDJqelLf63UgXNjR+hwRkXEaZMARWUZZsCk7P5bAo7CTbtiJadR0YSgHE1U1R0RkfBRwRFgcaur8bAyBJ9WJbG0MJ0ithS229jXov4VNQUdEROpQwJFRWybYVL3O2AJPCmFHVZ0429egn7Bz+cHdQdvWQEFHRGTIBhtwxrYOpyoNGMi0EWyq3lbfgUdhJ9Nn2Okq6EB7VR3oNuyEDDmg9TkiIkM22IBTxxgnqY1Rl6FmnpgCj8JOv8fW6SPoQLthp82gE/rAoarmiIgMkwKODF4swaZMLIFnrGGnz6pOH0EH0q/qtFHNAQUdEZGhUMBpYHXHbZt2avqkEdGzxR5q5okh8KQadlJtYes76EB6YSd0yAEFHRGRoVDAkUFJOdiU6TvwtDmRLXTYSb2Fra+gA+1WdaCdsBO6Za2g9TkiImkbdMDRoIHNhjxgYIjBpkxMgWcMYaePqs6e9dVOx0tPajvoQPj1OqrmiIjIpEEHHBm2MYWaefoMPAo79dSp6vRxHJ1JbbevQdiqTpvVHFDQERFJiQJOLpZJapM7YjKbgs18fQWesYadoQcd6LaqA8uFnTaqOaC2NRGRlCjgSBL6DDXbbzly9P8HTjqut+1oqo/AM6aws0xVJ9WgA3G3sLUZckDVHBGR2A0+4GgdTibV9TexBJt5p0FawafrwJNa2OmjhS21oAPxr9Vpq2UNFHRERGLX616Zmb3azL5kZp8sOf/RZnaLmV2af/1a19sYsyGPiN6233sLN9tvOVIaZBb9zORXKor7uov7fNva5q9QVtY2vpa1df+Wo1/L8LWVTYGnijrj5/esrzZqq716/eSjXyFcdeuuTZWdNky2r9VVBJ02XHFo5WjYERGRePRdwXkN8ArgdXMu86/u/sRuNkf61PfamtChJNVqT5cVnjaqOyErO320sNUdL91k4lohpfa1Zas5bVRyClqfIyISl14Djrt/0MzO6HMbpH99BxsIH27q3lbMoaerwDOmsBM66DRtWyuk1L525YHd0bWsgdrWRERi0ncFp4pvMrOPAdcDP+/un5q+gJmdD5wPcMe73PGYK6i6DieWSWpjEEOogW6DzTwphZ4uAs/Qw87Ygg6EDTtNQw50U80BBR0RicuBI1uXavdNTewB56PAPdx93cyeALwTuNf0hdz9QuBCgF3329XJnvPqjttq9cv3KaYBAwo21aXS4tZ24Ekl7HQRdOq0rUE8QQfCV3ViHUBQUNAREelPXHtKU9x9v7uv5/9/D7BiZu2uZpVW9Dk0YFJqAwBmiX2gQZsDC2IeULDMcIKqAwnW147vZBBBIeQwgkLooQSxDiAoaAiBiEj3oq7gmNldgBvd3c3s4WSB7KaeN0sqiiHQTIotCIQUa7Vn8jHQZmUHwlR3QlV2mraw+dpKa4MIIM6KDixf1Ym5ZQ1UzRER6VqvAcfM3gQ8GthlZtcCzwdWANz9lcD3As8ws8PAV4GnuHtce809iXlEtIJNPGJa25NaK1tfYaet9Tmw3MQ1aCfoQJj2tdhb1kBBR0SkK31PUXvqgvNfQTZGemkaNNCu2EINjDvYzBNLtaer6k6MYafPoLNsNQfaDzrQPOzEXs0BBR0RkbZF3aImy2t7wICCzXD0We1ps7oTY9hpM+h01bYG7QUdWK6qk0I1B3T8HBGRtkQ9ZEDiFcvQgGlthJvtew+xfe84d0L6GmjQ1qCC0EMKlh1QUGcoQZUhBND9IAJoZxhBYZmBBLEPIIAs5GgQgYhIWKrgSGUxBppCWzvek8FmUcg5sHMcOyldt7i11coWU2WnakWn7fU5EGdF56pbdw26ZQ3UtiYiEtKoAk7VdTiy2diDTcifGXIImv5btBF42mplCz2RrWnY2bp/S/KDCCC+4+ik0rIGCjoiIiGMKuDErsnxM+ZZpg0n5lADcQWbkNc/pAA0+Tcac3WnbtgZyvocYNMHSqHCzhiqOaD1OSIiy1DAmSH2SWptjoiOPdhAe+tsYjDUKlDX1Z3Yw05fQaePtrVCyKrOmEIOqJojIlKXAs4SVnfcVmsxb8zGGmwgnnBT1RCqQG1Xd7poZVsm7LQZdGJdn1MIFXTG0rIGCjoiInUp4IxYCqEGFGzqSq0KlGp1J0TYqRt0YlifA/EFnTFUc0BtayIiVY0u4GjQQDoUbNoTcxWo7cDTRnVn2bBTNejEcKBQiC/oqJojIiKTRhdwxqLtA3y2bcjrbFIQUxWoy3a20GEH6gWelbV0BhFAmIlrhVBBZ0zVHFDQERGZRQFHoqKqTTrK7tM2g0+X1Z2+WtnGvD4Hlg86Ywo5oKAjIjKLAk6J2CepLSu29TcKNsMxfZ+nGnjabGXrK+ik0rYGywWdMbWsFbQ+R0RkQzsHp+jRtuMOL7xM6KNsL1JlZ6KqNkdE96WtdjSFmzgUf4su/h7bbzly9Cu0bfv96NfS17VWvY10Za36877qsbR8beVo2Flkfe34WtMi96yvBv1waJk1k0XQaaIIOk0UQadrVxxaOVrREREZs8EFHElHGzuiCjZxmww7rR9UdSLshH6chQo7bQSdrfu3tBZ06ggZdBRy6lHIEZGxG2SL2j1P2LPUm1rqYh8wkFI72ta91Rdclzm8cxjHSmpDX+1ssbWy1W1dS+VAoRCudU0ta/VobY6IjNkgA47EaWzBpo3rGnpYmvxbDmHtTt2ws21tmIMIINzEtavXT9YAghq0NkdExmi0AWfMx8PpY8BASmOfQwaS0EJvW8yBaQjDCppUd9ocRDCU4+eomlOPqjkiMjajDThVVJmktrrjttq96WOiYBO3lKpLQ2pn6yvoDO1Aoarm1KNqjoiMhYYMDEws62/ammTVVjvaGMNNaMX92NX9mfKwgjoDCuoOIqhiSIMINICgHk1aE5ExGGwFJ7VBA1V2NlIYET3WdTZyrOn7t80KT6rtbG1UdFJdnwPNKzpqWatP1RwRGTJVcCSYlI5no3DTvb6qO11WeJqqW9Gpou7xc6p8yNLm8XNg+YqOqjn1qJojIkM16oDT9QE/Y9DGgIGUjmejdrQ4qJ1ttrba1lI6UCiwdMhpGnSuunVX46Bz5YHdjYNOnyEHdNwcEememf2smX3KzD5pZm8ys+MnzvtDM1tqkeaoA44sR+tsJJQuA0/s1R2tz8mMrZpz+cHdquaIyCiY2anAzwBnu/sDgS3AU/LzzgZ2Lnsbgw44TSfkTApx3IaudDVgoM1go3Y0gWG2s9UNO20EnTba1qB60OmjmtPUGFvWQNUcEenMVuAOZrYVOAG43sy2AL8N/GKIKx+1EMfDCTEq+vCJt1femShzcEf7IUdDBKRrXQ4rgG4ONlo8j6oMKKg7iGCIBwpdZgiBBhDUp+PmiAzPwSNbuxy+tcvMLpn4/kJ3v7D4xt2vM7PfAa4Gvgq8z93fZ2bPBt7l7l80q3eg7GmDruBAd1WcRW/UVd70F+1AVNkZWbRzU/fI6tNCHhNEpIkug2sXVZ2qQg8hgHqTGetUc6pSNWe+vqs5IiIN7XH3sye+Lpw808x2AucC9wTuBtzRzH4Y+D7gj0JsgPZWqfbJXohWtSohZ5EqIWeRGENOG5+Ut/1Jv/Sn6zVVKYac0GtzoL0hBHWkGHJSpXY1EWnJdwBXufuX3f0Q8NfAC4GzgCvM7PPACWZ2RdMbGEXAqVLFCTFRrUq7xaKQU6UNZFHIqdKismzIEZl23N79nd9ml0GnzWpOGyEH+g05UL2a02XI6UPKVRyFHBFpwdXAI8zsBMt60c4Bfs/d7+LuZ7j7GcCt7n5W0xsYRcAJJUSrWhVVQs4iVULOMlKp4ki3jtu7v7eg05W2go5CTjchR61q9SnkiEhI7n4x8Dbgo8AnyPLIhXN/qKbRBJxQVRytx9mg9ThSmA41Q6/mQDttazGEnNDHzAGFnIJCjohIxt2f7+73dfcHuvsPufuBqfOXKtVrD7UnXa3HaTvkhBa6iqN1OO0rCzOq5jS8zp5DDvQfcuoEHYWc7ijkiEgqRhVwYqriQDfrcapYJuSoijNuVQJMH0En9WrO2EMO1KvmKOR0RyFHRFIwur3T2ELOIikMHVDIGae6oUXVnJrXVzPk9DlGWiGnPoUcEZH2aM90CbGsx6mi7aEDIWnYQPyahhVVc2peV80D6/Ydcvo+Vo5CTncUckQkZqMMOF2NjS6MYehAzFUcrcMJK0RAUTWnxnUlFHKg/wOCKuSIiEi8e6UR6OoAoDCMg4CGDDmq4sQpZDBRNafG9SjkKOSU6DPkqIojIrEabcCpUsWpKqWhAzoIqDTVVhhRNafi9dxypJXhA9vW+j1Wjq+ttDJGWiGnGwo5IhKj0QacqrpsVauiq/U4TUOOqjjD1HYIUTWnxvVEcKycqvo+Vo5CTjcUckQkNqMOOF1XcSCd9TjLiHE9jtbhNNdl8FA1p+L1KORUvk6FnG4o5IhITOLbE41Q1SqO1uOEpypOv/oIHKrmVLyOCEJO3wcErUohpxsKOSISi8EFnO3HHa51+apVnK5b1VJfjxNjFUfq6SPcTN9+H0GnK0MIOaCQM49CjohIPwa5F3rm9htrXT7GVrUqxrIeR1Wc7vUdbiYNuZqjkFMuppCTossP7u4t6CjkiEjfBhlw2hK6VS2l9Th9h5wQtA6nmpjCTWHI1Zw+Qk7fBwStKpaQk2IVp6CQIyJjFM/eZ2BtVXGGuh6nzaEDIaiK040Yw82koVZzug450O+xcrbu3xL8WDkKOeUUckRkbAYbcFIRy3qcKoZQxZH0DXUIwdhCDoQ/IKhCTrk+1+WIiHRt0HueqVRxtB5HYhB79WbaEEdKK+SUU8hJM+SoiiMifdBeZ8u0HiecEG1qWoczW2rhpjDEak7sIafPY+Uo5CjkiIhUMfiA03cVJ6TUQ46qOHFKNdxMGtoQgphDDijkQPohp+ugo5AjIl3SHucS+mhVS33owLIhR8MGwhpCuJk0pGqOQk45hZww4UQhR0SGahQBp8/j4hS6DDmxDx2QOAwt3BSGVM0ZUsgJPUZaIUchR0SkzCgCDqTVqjb0oQN9V3G0Dme44WbSUKo5Qwk5oJAzi0KOiEh4owk4bWrj2DhDHzqg9Tj9GUO4KQylmqOQU04hRyFHRGTaqPYyU2pVqyKm9Thdt6tpLU4zYwo3k4ZQzekr5PR9QNAqFHIUckREJo0q4LSpr1a1rtbjtDF0QFWcbo013BSGUM3pI+RAv8fKUcipRiFHRGTD6PYwU6vidLUep6+hA8uEnGWqOGNbhzP2cDOp66ATupqjkFNOISdcyOnjeDkiIqFs7XsDhuT01X2V39h2ra5XfuNc3XHb3Ddk23Fo4Rv74RNvr7yTUObgjsU7LwdPtFq9+9I+hZvZjtu7nyM7T+zs9rbuvS1YsN6+99DSbZrbbzlS+wOGbfu90gcZ29aqV31X1qp9wLJ1/5ZKH9b42kql9t31teMrf4C0Z301aHtxm648sLv2B3llLj+4m3tvC3Nd81xxaIWzVpYP7iJS7sCRrUt9+JKa0VVwoN0qTp1WNa3HyfRVxRkDhZv5Uq7mpFDJCX2snK37t1T6oCaGSk5fVRwIV8mB7lrW1KomIiGNMuCkKLX1OF2GHJlN4aa6VNfmxB5yoL8DgvraSqWgo5CzmEKOiKRmtHuVKVZxYlqP08bQgaaaVnGGvA5H4aa+Pqo5ISjkzFc15FQNOqmFnNSGDyjkiEgIow04TaQQckIcHycUVXEkRQo51bUVcmIfPpBSyIH0Jqwp5IjIska9RxlqIWZsxroeRzKq3iyvy2qOQs5sCjkbFHJEROrR3mRNKVRxIO31OE1o2EBG4SasroKOQs5sCjkbYgs5XQQdhRwRaWr0AaftKo5CTn6ZGiGnyyrOkNbhKNy0RyFnsW37fZTHyhlryIFuqjkKOSLSxOgDThNtHPyziSEPHWgScsZcxYkh3BzZczNH9tzc92a0potqTmwhRwcEVchZRAcEFZEYDS7gHG/139hTreLAsIcOSDWxhJvJ/w896LQpppAD7R8rpyqFnA2hQk4q63JUxRGRugYXcIBGR15uc2x0XaGPmJ3q0IGuqjgpt6nFFm6mTx9q0Gm7mjO2kNPGAUGrGHvIgXSGDyjkiEgdgww4MapTxQGtxyloqlq5mMPN9GWGHHTaMqaQAwo5TSjkiIjMNti9xxirOHVDTh2prcdps11tDGtxUgk305cfYtBps5qjkDObQs6GGENOm0FHIUdEqhhswOlKTK1qXa3H6XrogKo4m6UYbqZ/VkGnOoWc2RRyNsQWcqDdao5CjogsMug9xy6qOHW12apWRapDB+qGnLpVnFTW4cQQbkIZctAJTSFntqGFnGUo5IiIbBh0wOlKaq1qqQ4dGLtYwk3oUDLEoNNGNUchZ7YhhZxlqjigkCMiUhh8wImxilNX6Fa1KmJcj9N2FSdmQw0309c9xKATkkLObAo5G0KGnBSGDyjkiMgsgw840Czk1NV2FWfo63HaCjlDMIZwM307Qwo6oas5Qwk5oY+Vo5CzIVTIgTQmrCnkiMi08e0tVhRbFacNXYacFMS4Dmds4Wb6NocWdEIZQsiB8AcEVcjZMLaQIyIyaTQBR1Wc2boaOtBXFSflNrUxh5vp2+97G0IJWc1RyJlNIWdDrCGnjaCjKo6ITBpNwGmiiypOKiFnEbWqDVNMwWJoQScEhZzZFHI2XHXrrtEMH1DIEZHCqPYSY6ziNKGhA/WkWMWJoXoTa5gYStBRyNlMIafcsiEHxjNhTSFHRGBkAaeJJlWcmMZGV5Xi0IE2qjgxrMNRuKlmCEFHIWczhZxyCjnVKeSIyOgCTqxjo1NpVYtt6EDVkJNKFUfhpr7Ug45CzmYKOeViCzkxDx9QyBEZt9EFnK6k2qqW4tCBoVC4WU7KQUchZzOFnHIxhRyIe8KaQo7IeI0y4AylitPEEIcODKGKo3ATTqpBRyFnM4Wccgo51SnkiIxTrwHHzF5tZl8ys0+WnG9m9odmdoWZfdzMHtr1Ni4jxioODHPoQMj1OF2vw1G4aUeKQUchZzOFnHKhQk5s63LaGCOtkCMSFzO7j5ldOvG138yek5/3LDP7jJl9ysxe1vQ2+q7gvAZ43JzzHw/cK/86H/iTUDc8pCpOk5CzyFCHDsRWxVG4aV9qQUchZzOFnHIhQg6MZ/iAiMTB3f/b3R/i7g8BHgbcCrzDzB4DnAs82N0fAPxO09voNeC4+weBeXse5wKv88xFwMlmdtduti6MJlWclFrVYhs6kBKFm26lFHQUcjZTyGnf0EOOqjgi0ToHuNLdvwA8A3ipux8AcPcvNb3Svis4i5wKXDPx/bX5aZuY2flmdomZXbLv5uoL4Luq4nQRctpqVesq5BzcsbiSc/BEW1jJCVXFabtNLYZwM1apBB2FnM0UcmYLVcWB4U9YU8gR6cyuYr88/zp/zmWfArwp//+9gUea2cVm9i9m9g1NN2Br0x+MibtfCFwIcL8Hba/2LpgrQk6dF9Ezt99Y+8X7nifsqf3mcfrqvlpvXkXIqfPpXxFy5r3pFiFn3ht4EXLm7Qwc2rF4p+LgjsU7KAdPtLk7O0XImbcjVYSceTttRcgJtYM46cjOE4H+g85xu05JYmc/lON2ndL3JiRt+95DQdo8t99ypNG6uW37vVK76ra1aq2vK2vVKsxb92+p9EGOr61U+lBofe34Sh8w7VlfrfTh1dXrJwer/F91665O1o/WcfnB3cEO1H3FoRXOWgkT1kVScvj247qsDu9x97MXXcjMtgHfBTw3P2krcArwCOAbgLea2de6e619e4i/gnMdcPeJ70/LTwuu7ovnmdtvrF3NuecJezo5CGjM1ZxDO7qt5izaieq7mlMEnT6NYaf/uF2njOL3LNNGSJe4hKzkhBKyXS0kVXJEovF44KPuXuxQXwv8db405T+AI0Cj0nLsAeddwA/n09QeAdzi7l9s68ZibVk7fXVfZy1ri4KO7Ti0MOgcPvH2YEFnkRDDBw7sXFkYdBRy0pR6sOm7wicbYm9VCynkCOlYKeSIROGpbLSnAbwTeAyAmd0b2AY0Kin3PSb6TcC/A/cxs2vN7MfM7AIzuyC/yHuAzwFXAH8G/FTb23TvbTc2qubU1dW6nDaDziKxrc1ZtppzeOfxrQUdhZxwilAzlN9nqFJbi5OCGKs4oegYOSLDYmZ3BB4L/PXEya8GvjY/fMybgfOatKdBz2tw3P2pC8534Jkdbc4m9952Y+11OVCvJF+EnDqfltVdlwNZ0GnyKeDqjts6W5sD83c0ipAzb6dl0docyIJOiLU5ba3L6fsT+5TX5CjQdCPUOhyJW6i1OFce2N3JIRaa0pockX64+1eAO02ddhB4Wojrj71FrVdjb1mDsG1ri4Q4Zk7Ias68nThVcuKgao3EaAjHxhERSZkCzgJqWcuEGEIQcm1ODCOlhx5yYg4NsW9fjDRooFzokdHSjrYO/qlWNZHhGVzA2W6NWvUWGvuUNeh+CMEiIYYQLFvNGXLIgfiqOQo2w9J0HU5VVdfh9CnmKk6MB/9si0KOyLAMLuAAnLVyqJWeWrWsZTRSerO2hg8o5GzcvoJNXEId9LOpqoMG+lS1TU3ioZAjMhyDDDiFtkKOWtbGPVK6TFshJ4ag00e4UKjJ9D14QjZLpU1tqFWcttrUCgo5IsMw6IAD8VRzYm9ZazvoLJLSSOkxr8vp4jYUbMal7Ta1qvocFx3zMXFERFI0+IBTiCHkQLwta9DuEIKuqzlVgs4iCjnHait4KNR0Y4yDBvpch9N3m1psVZxUqIojkr7RBBxop5ozpJa1Qgxta4vEMFJ6rMMHQgYRBZt09b0Op0+ptKlBXGOjUxg2UFDIEUnbqAJOIYZqTtOWtbqWCTkaKb2hSTVnyMMHlgklakOTENoYNKA2tXIxVXHaXodTUMgRSdcoAw60V82pq6t1OV0HnaGOlC49r4fhA32rG1IUaqRMLOtw2tB3mxqoirMMhRyRNI024BTUslZNn9UciGektELOseaFFlVrJAZtrMNpo02tahWnLzFVcURE5hl9wIE4qjmxt6xBOtWctkdKL1qXU2ZMIUehJqyQo6JDDhoY8zqcPjVpU4upihNCV21qoCqOSIoUcCb0HXIg/pY1aD/oLBLLSOlYhg/EdKwcBRtpKpY2tTbW4cTQphaT1NrUQCFHJDUKOFNCV3OG2rIGywWdeVIaKa3hAyL9aWPQQB19tqn1VcUZc5uaQo5IOhRwSvRdzemyZS1E0KkrxZHSZbQuR0QWURVnsxBVnC7b1AoKOSJpUMCZo41qTl1dtKxBv21ri8RSzZnXsqaQI2OVyjqcPg/4GRNVcURkDBRwKhhLyxr0E3RSGyk9L+Ro+ICkIOSggZCGvA6njtiPiTN2quKIxE8Bp6K+qzldtawVQgSdulIaKZ3K8AEZjpCT1CScOutwYmlTi6WKk2qbGijkiMROAaemsbSsFbo+SGhKI6VTGD6gkCND1/egARkvhRyReCngNBCymhN7y1oh1ra1RdoeKZ3C8AGFHGnbENfhDL1NLZYqTuoUckTipICzhDG1rEF/63PmiaGak8LwAYUcSUUs63DqSLFNLRYpt6mJSLwUcJYUuppTV9cta9D9+pyYRko3aVmLZfhALAcElTjEOmhANqiKkwZVcUTio4ATyNha1qCf9TmLdFHN6Xr4gKo5IjJkIao4fVPIEYmLAk5AfVZzmraspRZ0YhkprXU5IpuFXIdTt02tzqCBttbh9N2mNvYqTgxtago5IvFQwGlBSi1rEE/QqSOWkdJalyNd0qjo8anapraMECFHMgo5InHY2vcGDFURcpZ9sbv3thtrfzJ15vYbG5X8i5Cz7KdxRcip+6ZZhJyqn0QWIWfeDkARcuZ9YlqEnHmfwB7aUf5p7sEd5Z8KHzzRZn66XIScWZ9UH9i5Uvpp+OGdxwddO3Fk54naaRYZiD3rq42OQRaDKw/sbvQBnYhUc+TIcZ18YBILVXBaFqKa03RdTtM3i74rOm21rS2ybMta6XmRHxRUlZzx0qCBsPpuU1vGslUctaltUBVHpH8KOB0ItTanq5a1QopBZ54Qa3PmtaylfFBQhRwJoc91OHWkdDwcqNem1nQtTgyGMGygoJAj0i8FnA6FCDpdhxwIG3Sa6GsIwTzzQk6qwwcUciRldQYNyGxDqOLERCFHpD8KOD0IEXK6bFkr9HkMndDVHFh+pPQQhw8o5Igsp602tbFUcZYVS5taQSFHpB8KOD1JtZoD6QSdLqo5odflxHBQUB0QVGSzGNrUutJ3FWdIbWoi0h8FnJ6lGnKg/6BTVdvVHA0fkC6FnnoXetDAENfhpGjMVZzYqIoj0j0FnAgsW83pq2Wt0NcggpiqORo+ICJNtNWmBv0d/FNrcY6lkCPSLQWciPRVzYmlbQ3iCTrzNGlZ0/ABkfbFMmigzjocOdaybWqxrcMpKOSIdEcBJzIhqjlNjDHozLOomqPhAyLjkuo6HFVx4qKQI9INBZxIdd2yVhhK0Kli2ba1MQ4fEKki5DqcNsWyDqfNNrVUDXnYgEKOSPsUcCLWVzUHnl/goAAAIABJREFU4hpEAPWDTshqDpS3rY1t+IBCzjCFHjQgcbWp9VXF6VOsbWoi0g0FnAT0GXJiDDp1VA06y67NGdPwAYUc6VKbk9RiMsQqjtrUyqmKI9IuBZxELFPNWaZlDeILOm23rc0Tel1OqsMHdKycfoUeFT0kbQ4aSHUdDqRZxRlymxoo5Ii0SQEnMX1VcyD9oBOqmqPhAxsUcqRMKutw2hRTm9oylgk5fVZxUmhTU8gRaYcCToKWreYsS0FHwwcmKeRIymIZNADtt6mlePDPoVdxQCFHpA0KOAnrq2WtMKagU0bDBzIKOTIWKbepLSPVKo6IjJMCTuL6rubAMILOIqFb1jR8QGIT8yS1lAcNxNamlmIVZxkptKmBqjgioSngDETf1RxIO+hUqeZ0uS5HwwdEmmtz0EDbYp6m1lcVZwxtaqCQIxKSAs6A9DlpbdKQg86idTllxjh8QEFHUhk0ENM6nC6MrYqTEoUckTAUcAZo2UlrCjqL29Y0fKAaBZ32aFR0f2Jah6MqzmKptKkVFHJElqeAM1DLVHNg+EGniirVnDKpDx9Q0BEJp+46nDptak2piiMiQ6aAM3DLhBwYbtAJ1bbW9fCBFNflTFLIiVvMgwYkvD4O/qmJatWoiiOyHAWcEVi2mgMKOlDetqZ1OfWomjMuodbh1J2kVnfQQN11OGNuU+vLmNrUQCFHZBkKOCOybMgBBZ0mLWtalzObgo6MSYxtatBPFUeqU8gRaUYBZ2RCVHNg3EFnrOty2qKgI7K8lKo4alOrRyFHpD4FnJEKEXJgvEEndMiBcR0UdBYFHZF+pFTFGVubmog0o4AzYqGqOTDOoNPl8AEY9vCBSQo61bUxKrqNQQN9HQ8ntnU4bbepqYozXKriyNCY2clm9jYz+4yZXWZm32RmDzGzi8zsUjO7xMwe3vT6FXBEQaeCJtUcDR9YjoKOTKs7aECqUxUnfgo5MjAvB/7B3e8LPBi4DHgZ8EJ3fwjwa/n3jSjgyFFjCTpNhW5Z0/CBahR0RKpTFWcxhRyRfpnZScCjgFcBuPtBd98HOFC84Z8EXN/0NrYuu5EyPGetHAr2InrvbTcGezM5c/uNS31yVyhCTpM31yLkzPrUctfqeumnoKs7bivd8bAdh2a2ohQhZ1ZbSxFyZrXIHNxR3mpz8EQrbds5cNJxMz8hL0LOrDajIuR0cfyUyZDTRmtWitoIfm0E13lBWebztZW5Fd+Q9qyvzv2wpszV6ydXHrcvIj253TqbzgjsMrNLJr6/0N0vnPj+nsCXgb8wswcDHwGeDTwHeK+Z/Q5ZEeabm26AKjgyU8zVnFDaqOYsWpfTVcvaonU5KbasTSqqOmOu7CjciDSnKo5Iq/a4+9kTXxdOnb8VeCjwJ+7+9cBXgF8GngH8rLvfHfhZ8gpPEwo4MleokANE27LWNOh03bLW1SjpRS1rfU1ZKzO2sNPW75pCuJkXwGVDSm1qTYWo5otIb64FrnX3i/Pv30YWeM4D/jo/7a8ADRmQ9oSu5oQSQzUnlilri0ZJdz1lrY+gA8MPO20FmxTCTRPzHt8iItIPd78BuMbM7pOfdA7wabI1N9+Wn/btwGeb3oYCjlQWa8taqgMIFrWsdTWAYFHLWpMBBNBv0IHhhZ1UqjYQR7gREZGoPQt4o5l9HHgI8GLgJ4DfNbOP5d+f3/TKFXCklhhb1iBcNWfZlrV51ZwyTao5i1rWujow6LyWtULfQQfSDzuphJsqj4eYzXt+SDv6PB5OyutwRFLn7pfm63Me5O5Pcve97v4hd3+Yuz/Y3b/R3T/S9PoVcKQ2DSCYTwMIZiuCjsJOPSmFGxERkRgo4EhjMVZzNIBgs1gGEEyLIehA/GFH4WbiNjRgQEREKtI7hixFAwjKaQDBYrEEHYgv7CjcSEpmHRssBam2qWlUtMh8CjgSRKwta6lWc4YwgCDVoAObw07XgWfMY6BD0gQ1EZHxUsCRYFTNKTfGAQRQfwc4lnU607oKO21d/9jCjYiIjJsCjgSnak65GFrWoPsBBE12hmMMOtBe2NExbiQGe9ZX+94EEZGlKeBIK2IcQADxVHPKxN6yBs0GEMDwgg6ECzsptaT1EW40YEBmufJAmPUzqa7DEZFyeteQ1oxlnHSToDPWljVo/ul/rO1rhaZhJ6VwI5vN+5BARET6o4AjrYuxmhOyZQ2WCzqzDOGYOW1UcwoxBx2oHnYUbuKhg3xKajRJTaScAo50YgwDCKBZ29pQj5kD7VVzCrEHHSgPOwo37dEENVhfi/t5ISLSJgUc6VSsLWt9V3OGfsycNqs5kEbQgXansSncSKquunVX35ugdTgiA6OAI52LsWUN0q3mxDKAYF7LGlSr5oQKOimEnZCGHG40YEBEROrSO4f0YgwDCKCdak6ZIQwggHA71mMIOhoDLSIiciwFHOlVjNWc0C1rELaaM/QBBBB2HPFQg86QxkDLMFy9fnLfm7AUtamJDIcCjvRuTAMImlRzygx9AAGErSQMKeioaiMiIlJOAUeiEWvLWt/VnCEMIIilmgPpr9NRuJlPE9RkTDQqWmQ2BRyJSszVnBgmrZWJfQABhKnmhN4RTy3ojC3caMBAxtfi/Rv17coDaisTkWPp3UOiFGM1B/pvW0tlAEFb1RxoZ4c8haAztnDTJR3kUwpahyMyDAo4Eq0YBxBAPG1rs8QygADSq+ZAvO1rCjfSpT3rq31vgojIUhRwJGqxjpOG/tvWhjCAIMZqTiGGoKMx0CIiIvUp4EgSYq3mQDtBp6rUBxBAvNWcQl9Bp63bTCXcaP3NuFx1666+N+EotamJpK/XdxAze5yZ/beZXWFmvzzj/Keb2ZfN7NL868f72E6JQ6wDCAohg04X1ZyUBhCMLeiM/Rg3TcONJqiJiAj0GHDMbAvwx8DjgfsDTzWz+8+46Fvc/SH51593upESpVhb1gqxVnPKpDCAAKrv9La9E9/2Op2xt6SlUrmZF+ZFuqRR0SLHWvhOYmbPMrOdLdz2w4Er3P1z7n4QeDNwbgu3IwMUc8saxFnNGcoAghiqOYXQQUfhJo1wk4r1tbiGZYiIdGVrhcvsBv7TzD4KvBp4r7t7gNs+Fbhm4vtrgW+ccbnvMbNHAZcDP+vu10xfwMzOB84HOO3ULQE2TVJQhJwQn14VISd073URckIcq+GeJ+yp3Kd++uo+rl4/eeZ5u1bXS6ckre64beZOURFyZh2Powg5W/cf+9wrQs7K2rG3VYScbbPOy0POtv2zX2oOnHQc2285MvO8TZfbucL2veHCcJkimGzdW14Nq3odISnciDRz+cHdrXz4JdIXu332+/RQLXxHcfdfBe4FvAp4OvBZM3uxmZ3Z8rYBvBs4w90fBLwfeG3JNl7o7me7+9l3upPeJMcm9moOhKvo1KnmxDKAAPobJw3drj1pWtFRuOnvdVvHwAmr7IMVEZEuVXpXySs2N+Rfh4GdwNvM7GVL3PZ1wN0nvj8tP23ydm9y9wP5t38OPGyJ25MBi3mc9KQ+2tZiGkDQ18FBoZ+gsyi4aAy0KjeyvBAVchEZliprcJ5tZh8BXgb8G/B17v4MsrDxPUvc9n8C9zKze5rZNuApwLumbvuuE99+F3DZErcnIzCmag5UH0IQywAC6LeaA93v/JeFmLGPgYaw4UYT1CQ0jYsWSVeVNTinAN/t7l+YPNHdj5jZE5vesLsfNrOfBt4LbAFe7e6fMrNfBy5x93cBP2Nm30VWNbqZrEVOZK7Qa3PaepMLtT6nCDlV1ueUrc0pQs6stTlFyClbmzNrXQ5kQaes3/fQjtnrcqD52pxiZ7nq2hygk/U5hcl1OmOv2oAqNyIi0p4qa3CePx1uJs5bqqLi7u9x93u7+5nu/pv5ab+Whxvc/bnu/gB3f7C7P8bdP7PM7cm4pNCyBmHX51TRpGUNyqs5i1rWmoyThmFWc0DrbUDhRiQ0jYoW2UzvMjJooVvWYg86VdfmpDSAYEhrc9qQ2rYr3KShbNKiiEgK9E4jgxdyAAF0E3SWFesAgtirOcmFhdS2V+FG5qg6Br9LWocjkia928hohAw50N4QAui+mlMm9AACaG+cdIhqDqQTGlLZzkKb4UYDBkREZJICjoxKitWcEEFnkT5a1ppUc+a1rMF4qjkxb9ssqtyIiEiX9K4jo9RGNSfmoNNmNadpyxr0c3DQ1Ks5MW7TPDGHGx3kU6pQm5pIeuJ95xFpWeiQA8MIOl0eMwfaqebMa1mDNKs5sWxHHTGHGxERGS69+8iohW5ZK8Q+iGCZas6ilrUhV3P6ChipBRtQuInFrONXtW3WsbbatuzxxIZAo6JFNugdSIR2qjkQ9yCCqtWcMn0MIOi7mgPdhw2Fm/jMC9wiItK/Yb8LidSQcjVn2aAzzzIDCEKPk4b+x0lDd9UchZvFNEFNuqB1OCJpUcARmdJmNSfWoKNqTv2WNWg3gCjciIiINKN3I5EZ2qrmQPxBZ55lBhComlPvOlOjcCMiIrHQO5LIHG2FHIh3EMEy1Zx5LWvQz8FBu6zmhAgmKYYbkVCuunVX35tQSm1qIulQwBFZoM1qDsQ7iKDNlrXQ1Zx5LWvQXTUHmgeUFMdAF1Ks3ugYOIvtWV/texOkJk1SE8mk964k0pPUqzlNgs6iak7TAQTQTzWn9LyeqzmpBhtIM9ykyNfSfYyIiHRN70wiNXRRzUk16JRZJuSErubMa1mDfqo5CjfNaYKaiIjMooAj0sCYg06ZNsZJw7CrOQo3ImnROhyRNOgdSmQJQwg6dfVVzSmTUjVnMtAo3IiEdeUBhQ8RyehdSiSAMQ4i6LqaM69lDdqr5pRpUs2BtIcJgMKNVHP1+sl9b4KIjJjeqUQCSr2aUzfojKGaM69lDca1wz+m31VERNKldyuRwIbQttYk6JRZFHLGXM1JydB/P5GqYl+Ho1HRIgo4Iq0ZW9CZV82Z17IG7VVzyqiaU0+Mv5cmqFW3vnZ835sgItKp+N61RAZmCEGnjpiqOfNa1iCeg4PGbGi/D+ggnyIifTOzk83sbWb2GTO7zMy+ycxOMbP3m9ln8393Nr3+4b1ziUSqzZADcQ0iSK2aU6ZpNWcoLWtD+B0kPVfduqvvTVgo9jY1kQS8HPgHd78v8GDgMuCXgX9093sB/5h/34jevUQ6NIRqTt2gU2aZkBO6mjOvZQ3GWc1JedvbNO9xIiIii5nZScCjgFcBuPtBd98HnAu8Nr/Ya4EnNb0NvYOJ9CDlsdJQr22tjXHS0E81p/S8xKo5xTbN+5Lw5j32REQGZJeZXTLxdf7U+fcEvgz8hZn9l5n9uZndEdjt7l/ML3MD0LhUurXpD4rI8s5aOdTaxJsi5LTVSlGEnCoH1ytCTlnryemr+0qPm7FrdZ0966szz1vdcVvpAmrbcQhfm33fHj7xdrbu3zLzvCLkrKwde14RcrbNOA+yoLNtv88878BJx7H9liOzfzCAMYWSZQYMpLr+Zl5oFxFZxI7Mfl9ryR53P3vO+VuBhwLPcveLzezlTLWjubub2ew31ArG844oEqmU1+ZA2GpOmWUqOakPIKhSbVG46Uaq7WnzniMiIj24FrjW3S/Ov38bWeC50czuCpD/+6WmNzCed0WRiCnkZNqYsgb9tKyVqRpYxhZcutBn9UbtaSIiGXe/AbjGzO6Tn3QO8GngXcB5+WnnAX/T9Db07ikSibGtyxnKlLXS8xYcM0fqG2v1ZmztafM+BBGRwXgW8EYz+zjwEODFwEuBx5rZZ4HvyL9vRGtwRCIzlnU5kO3IjHldjnRnmerNsuFG1RsRkc3c/VJg1jqdc0Jcvyo4IhFSy1pmDOtypJoxDhbo27zn2CLznrsiIm1TwBGJlEJOZgzrciRefVdvlm1P04ABERkjBRyRiA0h5FQNOkNal1MWdLQupxlVb0REpA4FHJHIdTF8YCjVnDLLhJyyoHNox/IDCOZ9Sf9SHQstIjJ2GjIgkog2hw9AFnTaGj4AWcjpe/gAMHMAQRFymg4gKDt42sEd5cMHFmkz5KQ09CDl6o2GC4iI9EMBRyQhQwg5UG3K2qKQA8wMOkUlp+spa22EnLakEp7GOha6MLbx0CIioSjgiCQm9ZAD1as5RbtaKqOk54UciC/otCGW9jpVb5YbMLDMBDURkb5pDY5IglI/KChoXY7EK4bqjYiINKeAI5KwIQwfqDNlrcwyIacs6Mw7Xg60M0pawuj7Pg5RvVF7mohIcwo4IolLfZQ0VK/mLAo5ZUFn3vFyoL9R0n3viMuxVL1Zng7yKSJ9U8ARGYCxhZzYWtbKLGpZA4Wd0HQ/ioiIAo7IQAxlXU7fLWtl5rWszVuXA9WrAgo6y1n2vgtRvYmlPW2ZAQMiIqlTwBEZmNTX5UC4lrUy81rW5q3LgXZa1qapqiMiItKcAo7IAKllLTNvXQ7007JWl8JONUOp3sRAI6JFJHUKOCIDNZSQE3PLWpll1+WUUdgZvrFPT5v3PBYRqUoBR2TAhrAuB7ppWSvT97qcMgo7G2Ko3ojEou0Pt0RSoIAjMgJal5OJcV1OiJ1rhZ3+xdSepgEDIjJ2CjgiIzGmlrWU1uVAuKAD4ws7Q6rejL09TUQkFAUckREZQsiBuFvWyixqWYOwQQfGF3b6ElP1pm86yKeIxEABR2RkFHI2LBNyyoLOvHU5UG1nOHTQgWGGnSFVb2KhCWoiMgQKOCIj1MXwgdha1srMa1mbty4H2mtZK7S1Az7EsNOXUNUbtaeJiISjgCMyYmOq5sxblwNxtqxBO9WcSamGHVVvZtOAARERBRyR0esi5AxlylqZpiEH4gk6kE7YiX37RESkXwo4ItLJcRNSalkr09a6HIgr6EA6YaeJUPef2tNEROKkgCMiQPvrciC+lrUyfazLKcQWdCCuoKPWNBERWWRr3xsgInE5a+UQVxxaae36i5Bz+cHdrd0GZCHnygPzb6MIOVfdumvm+aev7uPq9ZNnnrdrdZ0966szz1vdcRvra8fPPM92HMLXFt+/h0+8na37tyy83KEdsLK28GJBTIaLbR3dZqyGOBpaE9REhstuH9frtio4InIMtaxtaGtdTqrVnEIfLWxDrN6oPU1EJDwFHBGZaSghB8K0rJWZ17I2b10ODCPowLDX68wSa/Wm7wlqOsiniMRCAUdESnW1LieWas68UdLz1uVA82oODCfoQHthZ4jVGxERaYcCjogspGrOhrZCDlRvV0oh6MD4Kjt1qT1NRKQdCjgiUolCzoa2Q06doFNFDNWLZcJOTNWbWNvThmDec05EpA4FHBGprKuQE1PLWpm21uUUhtS2Nq1O2FH1R0RE6lLAEZFaugg5EE81p811OWMPOtB+G1us1ZvQ7WnLDhjQiGgRGRIFHBGprYvhA9BtNWeRNlrWoPqO6dCDDhwbdlS9ERGRJhRwRKSxIVVz2m5Zm6dqNQeGN4igTIhwE+PvJSIi7VPAEZGlDCnkQLWWtTLzWtbmrcsphG5bg7QGEcQs5vY0ERHZTAFHRJbWZciJoWVt3rocWK6aA1qfE8JQfo9U6CCfIhITBRwRCaKrdTkwjJY1BZ10xD4aetkBAyIiQ6OAIyJBDSnkwPIta/NUnVylQQT1pLjNfdIENREZGgUcEQlujC1rZaqMkg5ZzYHxDCLoQujqjdbfiIi0TwFHRFoxtpa1ZdblQH9tazCsQQQpbKNIW7p6zRWJnQKOiLRqSCEH2m1ZA63PkXq0/kZE5FgKOCLSOrWsbag6bUpBp77Q26P2NBGRNCngiEgn1LK2YdG6nEkaRCAiIlKPAo6IdErVnA1Vg44GESwWe/WmDWpPExGZTQFHRDo3xGrOPPNCDvQXdMY6iKAPsbanhRgRrYN8ikhsFHBEpDdDCzmLWtYWiT3oxFrNGWP1ZmiqPD9ERKpSwBGRXo2pZW3RKOmCBhGIiIg0p4AjIr1Ty9qxNIigmhQCVBvtaVp/IyJSTgFHRKIxtJATqpqjQQTl1xea2tNERNKngCMiURlTyxrEH3Q0iEBERFKjgCMi0RlbyxpUX2Qde9DpqpqTSvUm5va0EBPURERi1GvAMbPHmdl/m9kVZvbLM87fbmZvyc+/2MzO6H4rRaQvQws5oao5oEEEIiKSLjP7vJl9wswuNbNL8tN+28w+Y2YfN7N3mNnJTa+/t4BjZluAPwYeD9wfeKqZ3X/qYj8G7HX3s4DfB36r260Ukb6NrWUNwretwbAGESgQySxVnlsiEpXHuPtD3P3s/Pv3Aw909wcBlwPPbXrFfVZwHg5c4e6fc/eDwJuBc6cucy7w2vz/bwPOMTPrcBtFJAJjbFmD+NfnQL1BBFUsCi9thZuxtaeFooN8ikgo7v4+dz+cf3sRcFrT69oaZpMaORW4ZuL7a4FvLLuMux82s1uAOwGb3vHN7HzgfIDTTt3S1vaKSM/OWjnEFYdWWr+de2+7kcsP7m71Ns7cfiNXHgh7G6ev7uPq9cUV/V2r6+xZX114udUdt7G+dvzCy9mOQ/ja4r/L4RNvZ+v+xa/Rh3bAytqxp7VhrMEmpvU3KR7ks4sPQkRCstth237v6uZ2FW1nuQvd/cKpyzjwPjNz4E9nnP+jwFuabsAghgy4+4Xufra7n32nOw3iVxKREmOt5FTVZ8taFXUrOW2t0anaOldHnYlzVdWppnVN1Zu4dPXaKFLBnmK//P+3d//Bltd1HcdfL5fdRVwUdA0QUJGFkizBITKpJLREdEBNDGsUSyIdmdCcMZIZnRxnImusNFMZYMSGRPLnlhghyJDjQGzOKj82dcFU1nVpEcENwf3x7o/zve7h7jn3nh/fH5/P5/t8zNzZe889nvP1fjm73+d9f77fU30sjhdJ+tWIeI4Gp6q8yfavL3zD9kWSdkm6ctYN6LIGtkg6cujrI6rbRt7H9n6SniDpvla2DkCySjovh8gZj7BpJmxKnN60ef4N0xtgfhGxpfrzXkmf1uDUFdl+naSXSvq9iJh55NRl4Nwq6RjbR9leJelsSesX3We9pHOqz18p6YZ5/s8CKEebv61M5YCmiaU8qUdOXQibejG9ATAr24+zfeDC55J+S9Lttk+T9DZJZ0TEQ/M8R2eBU51EdL6kayVtknR1RNxh+122z6judpmkJ9neLOlPJO1zKWkA/VXKxQem+e1z3ZeRlsqOHMLm0VKa3tSF6Q2QnUMkfcn2VyX9p6TPRcS/Sfp7SQdKuq66fPSHZn2CLi8yoIi4RtI1i257x9DnD0s6q+3tApCXEi4+MM1FB446YLu+9dDaZe836UUHpPQvPDCtvl48oA11TW9yu7gAcQPUIyLulvTsEbevq+s5OCMfQBGY5IzWxFKilCc5TGzGK3F6gwEuMAA8GoEDoBglRM406o6caQ6AU4scwqYdqU1v2lqelsprHsBkCBwARcn9CmtNHbCVGjmEzWSY3gDoEwIHQHFyv8JaE0vVpGYiZ1J1Rw5h077UpjdtYXoD5IfAAVCk3K+wlkvkTHPwXkfkEDbTK3V60+bV01LG+TfAvggcAEUjcvaVY+QQNt1iegMgJwQOgOLlHDnTyCVyJrUQNYTN7OqY3qT4pp5tTG+6fj0DmB2BA6AXco2caQ/kcoicJt5PZtLn7UvYSOktTcttegMgXwQOgN7I9QprRM58+hY2dUpxetOGXKY3nH8DjEbgAOiVXK+wRuRMr89hk9r0pk5cXADAcggcAL2T+xXWJtXXyOlz2NSpzulNTsvTcpneABiPwAHQW7lFTtO/uc49cgibgZKnNwAwCQIHQK+VHjnT/uY8x8ghbOqX6vSm6chnegOUgcAB0HtEzqPlEjmEzb6Y3swut7jhAgPAeAQOACjfK6xNqqTIIWya1dfpDYByEDgAUMnpN6KzHOzlHjmEzdKY3swut+kNgKUROAAwpK0rrNVxQNWXyCFs2pPq9AYApkHgAMAIRM5eXUYOYTOZ0qc3TS5Py3F6k9O0GegCgQMAYxA5e3UVOXUpNWzq1MfpTY5xA2B5BA4ALIHflE4vpcgpPWzWrtnB9AYAFiFwAGAZTUdOaVMcqfvIKTVsFoKm7rCpc3qTC6Y3QLkIHACYAJEz0MSBcJ0hUlrYNBU0TcpleVqumCoDyyNwAGBCTV9hrbTImeaAfN4oKSVsugialKc3TS1PY3oDlI3AAYAplfgb1Fwjp4SwyW1CsxSmNwBSsF/XGwAAOVq3cqc271xZ++Meu2qbvvGTQ+Z6jKNXb9Ndj0z/GEcdsF3femjtxPd/6pof6js7Dlr2fmvX7ND2HWum3p6l5Bw1KYVMytObpjC9QR95t7T6gT1db0ZrmOAAwIyamuR0tVRN6n6Ss1y45DixyfE8mlnUPb1pYnkacQP0A4EDAHMgctqJnJzCJpeg6eP0JnclLo8FmkDgAMCcSjzoSCVycgibXIJmWN1xw/QGQEoIHACoQRNXWOtyijOLtperdSm3oAGAPiFwAKBGJUXOLL+Vb+LqainIcUozTurTGwCYF4EDADUjcvKPnJKCJjcsTxutxKWwQFMIHABoQEkHI32InL4EDRcWANAHBA4ANKTOyOn6fJwSI6cPQdM0lqcBSBGBAwANInLSiZy+TGnGyWF6w/I0AHUgcACgYXVeYY3ImVzfg6ZpTG8ApIrAAYCWEDnLmydECJrx+jq9KUVJ5/QBbSBwAKBFfT5QqTtyCJru5DK9YXka0E8EDgC0rI7IyXGKI80fOQTN9HKY3gBAnQgcAOgAkbO8xRMaoiYNTUxvWJ4GoE4EDgBkrPTIwXz6/HMuZXlan5e1ArMicACgI6UcuORyPgbmx/QGQA4IHADoUAlL1aRmr6yG2fDzBdBXBA4AdIzIAepVyvI0ALMhcACgEEQOFjTxM2V5WvtKWcYKtI3AAYAElHQgQ+QAALpE4ABAIkpZqjYrIqceuUxvmsLyNAAEDgAkpJTI4fLR3cjp58fyNABNIXAAoEBEDupO+ol/AAASt0lEQVSS0/QGACQCBwCS0/fzcSQiZxY5/cyamt6UtDytpL8HgLYROACQoFKWqknzRc7iD7SL6Q2AHO3X9QYAAEZbt3KnNu9cOddjHLtqm77xk0PmeoyjV2/TXY/M9xhHHbBd33po7VyPIbUzpfjOjoMaf466EX8AsBeBAwAJI3LaR0QNNDW9YXkaAEmyvULSBklbIuKlQ7e/T9IfRMSaWR+bwAEATKRPkdM0Ji5YCuffoCcukLRJ0uMXbrB9oqSD531gzsEBgMSlcj4OkCr++wbyYvsISS+RdOnQbSsk/ZWkt837+ExwACADKS1VkzTXJGdh6ROTnHTltjwNwNIeszu0+v7WJoNrbW8Y+vqSiLhk0X3+VoOQOXDotvMlrY+Irbbn2gACBwAyUVfkSOKcHIyVW9yUNr1heRoKsD0iThz3TdsvlXRvRPyX7VOq254i6SxJp9SxASxRA4CMrFu5M5kla11eQhrNIG66RdygJ06WdIbt/5F0laRTJd0haZ2kzdXtB9jePOsTEDgAkCEiB3UjbgC0ISL+LCKOiIinSzpb0g0RcXBEHBoRT69ufygi1s36HAQOAGSqtMghdLqTW9yUiOkNUB8CBwAyllLkMM3JU45xw/QGKENE3Dj8HjhDt8/8HjgSgQMA2avjvJxjV21LapqDdhA3aWB6A9SLwAGAQqQ0zZkXkdM84gZAqQgcAChIaZHDuTnNyDFuSsX0Bqgf74MDAIVJ6U1B532vnAXjDsh5H53p5Ro3JU5viBugGQQOABQolTcFXTjorSt0Fht1sE70jEfcAOgDAgcACrXw2+HSpjnLYdozGnGTFqY3QHMIHAAoXIlL1mbR5/AhbgD0CYEDAD1A5IxX+jK3XOOmZExvgGYROADQEylFjtTceTl1KGXak3PcML0BMCsCBwB6JJWLD0hpTnOWk1P4EDdpYnoDNI/AAYCe6evFB5qU2jI34iZNxA3QDgIHAHoqpSVrJUTOYl1Ne4gbAH1H4ABAjxE57WsyfHKOm9IxvQHaQ+AAQM+lFDlS2hcfaNK8y9xyjxumNwDqQuAAAGo5L6fPFx9oyqTTHuImbUxvgHYROACAn0ppmkPkjNdU0AwjbgDk6jFdbwAAIC11/La5joPWo1dv49yPjvBzrw/TG6B9BA4AYB+pRI60N3Q46G5Hmz/n0qc3xA3QDZaoAQBGSulNQRcsPvhmGVu9iBsAJWCCAwAYa93KnUlNcxZjulMf4qZeTG+A7jDBAQAsK5WLDyyF6c7siBsAJelkgmP7ibavs/3N6s+Dx9xvt+2N1cf6trcTALBXypOcUZjuTIafT/2Y3gDd6mqJ2oWSro+IYyRdX309yo8j4vjq44z2Ng8AMEpukbOA2Bmt7Z8H0xsAbehqidqZkk6pPr9C0o2S/rSjbQEATCHFiw9Mg6VsA8RNM5jeAN3raoJzSERsrT7/vqRx/7rsb3uD7Zttv2zcg9k+r7rfhvvu21P7xgIAHi31iw9Mo4/THeKmGcQNkIbGJji2vyDp0BHfumj4i4gI2zHmYZ4WEVtsP0PSDbZvi4i7Ft8pIi6RdIkkHf/sVeMeCwBQsxwuPjCNPkx3+hRyAAa8a4/2u//hrjejNY0FTkS8cNz3bG+zfVhEbLV9mKR7xzzGlurPu23fKOkESfsEDgCgO6VFzrDhGCghdrqIG6Y3ANrW1RK19ZLOqT4/R9JnF9/B9sG2V1efr5V0sqQ7W9tCAMDESlmutpThpWw5TkGIGwB90dVFBi6WdLXt10v6tqRXSZLtEyW9ISLOlfRMSR+2vUeDELs4IggcAEjUQuTMM83p8uID0yptulO3PsUN0xsgLZ0ETkTcJ+kFI27fIOnc6vMvS/qFljcNADCnkpesjZP6uTtcVABAn3S1RA0AULA+LFlbSkpL2VLYhpIxvQHS09USNQBA4ep8vxwpj2Vro3S5lI3zbppF3ABpInAAAI2pI3IWlBY7UrPBQ9wA6CsCBwDQqDouPrBYThcjWEpT0x3ipnlMb4B0ETgAgFbUOc1ZUMJUZ0Fd0x3OuQHQdwQOAKA1TUTOgpJiR5ptutNV3DC9AZASAgcA0KomI2dBKUvYFkwy3SFuAGCAwAEAtK6NyJHKm+osWDzdIW7aw/QGSB+BAwDoRBMXH1hKH2KnTcQNgFQROACATrU1zRlWauwAAKTHdL0BAAB0+ZvxY1dt6+U0Yh59/HkxvQHywQQHAJCELiY5w5jqTKaPcQMgLwQOACAZbZ+XMw6xM1pf44bpDZAXlqgBAJKT0gElS9gAIC9McAAASep6ydpifZ/q9DXyUoptAJMhcAAAyRo+uCR2ukPcAMgJS9QAAFlYt3JnkgecpS9hK/n/G4AyMcEBAGSFqU57+hw3KcY0gMkQOACAbBE7AIDFCBwAQBFSj53cQofpDYBcETgAgOKk8n46w3Ka6vQ5bgDkj8ABABQr9amOlF7s9D1umN4AzbK9v6SbJK3WoEU+ERHvtG1J75Z0lqTdkj4YEe+b5TkIHABAL6QeO6mFTh8RN0ArHpF0akTssL1S0pdsf17SMyUdKennImKP7Z+Z9QkIHABA77CEbfltAIAmRERI2lF9ubL6CElvlPS7EbGnut+9sz4H74MDAOgt3lunm+dLUYr/HQCZWmt7w9DHeYvvYHuF7Y2S7pV0XUTcIuloSb9T/W8+b/uYWTeACQ4AoPfWrdyZ1DRnwbGrtjU+zel72EjEDXpg92495v4H23q27RFx4lJ3iIjdko63fZCkT9t+lgbn5DwcESfafoWkyyX92iwbwAQHAACle5DbVIAwtRlIdb8DfRARP5T0RUmnSbpH0qeqb31a0i/O+rgEDgAAlVQPdusMEcIGQJdsP7ma3Mj2YyX9pqT/lvQZSb9R3e35kr4x63OwRA0AgCGlLlcjavaVatAChTtM0hW2V2gwbLk6Iv7V9pckXWn7LRpchODcWZ+AwAEAYJEUr7ImzRY5hM1oxA3QjYj4mqQTRtz+Q0kvqeM5WKIGAMAYKR4ET7PEjLgZLcX9CqA+BA4AAEtI9WB4qXjhPJvxUt2fAOpD4AAAsIxUD4oXRwxhs7RU9yOAenEODgAAE0j54gNYHnED9AcTHAAAJsRBcn7WrdzJfgN6hsABAGAKHCzngbAB+ovAAQBgShw4p4uwAUDgAAAwAw6i00LYAFjARQYAAJhRqhce6AuCBsAoBA4AAHNYOMgmdNpD2ABYCkvUAACoAQfdzWMZGoBJEDgAANSEg+9mEDYApkHgAABQIw7E60PYAJgFgQMAQM04KJ8PYQNgHgQOAAAN4AB9eoQNgDoQOAAANISD9ckQNgDqROAAANAgDtzHI2wANIH3wQEAoGG8IeheBA2ApjHBAQCgBX0/sGdaA6AtTHAAAGjJwgF+n6Y5RA2AtjHBAQCgZX046GdiA6ArBA4AAB0o9eCfsAHQNQIHAICOlBQChA2AVBA4AAB0KPcoIGwApIbAAQCgYzkGAmEDIFVcRQ0AgATk8l45RA2QoV27tWf7D7reitYQOAAAJCLVyCFqAOSEJWoAACQkpZhgGRqAHBE4AAAkpuuoIGwA5IwlagAAJGghMNpcskbUACgBExwAABLWRnQwsQFQEgIHAIDENRUfhA2AEhE4AABkoM4QIWwAlIzAAQAgE/NGCWEDoA+4yAAAABmZ9r1yCBoAfcMEBwCAzEwSLUxrAPQVgQMAQIbGxQthA6DvCBwAADI1HDKEDQAMcA4OAAAZI2oA4NGY4AAAAAAoBoEDAAAAoBgEDgAAAIBiEDgAAAAAikHgAAAAACgGgQMAAACgGAQOAAAAgGIQOAAAAACKQeAAAAAAKAaBAwAAAKAYBA4AAACAYhA4AAAAAIpB4AAAAAAoRieBY/ss23fY3mP7xCXud5rtr9vebPvCNrcRAAAAQL1sH2n7i7bvrHrggur2423fbHuj7Q22T5r1Obqa4Nwu6RWSbhp3B9srJH1A0oslHSfp1baPa2fzAAAAADRgl6S3RsRxkp4r6U3VMf57JP15RBwv6R3V1zPZr5bNnFJEbJIk20vd7SRJmyPi7uq+V0k6U9KdjW8gAAAAgNpFxFZJW6vPf2R7k6TDJYWkx1d3e4Kk7836HJ0EzoQOl/Tdoa/vkfTLo+5o+zxJ51VfPrL28C23N7xtaM5aSdu73gjMjP2XL/Zd3th/+WLf5e1nu96ASTy4575r//3/Prq2pafb3/aGoa8viYhLRt3R9tMlnSDpFklvlnSt7b/WYJXZ82bdgMYCx/YXJB064lsXRcRn63yu6od2SfW8GyJi7Hk9SBv7L2/sv3yx7/LG/ssX+y5viw7kkxURp3W9DYvZXiPpk5LeHBEP2n63pLdExCdtv0rSZZJeOMtjNxY4ETHTBg3ZIunIoa+PqG4DAAAAkCnbKzWImysj4lPVzedIuqD6/J8lXTrr46d8mehbJR1j+yjbqySdLWl9x9sEAAAAYEYenIR/maRNEfHeoW99T9Lzq89PlfTNWZ+jk3NwbL9c0vslPVnS52xvjIgX2X6KpEsj4vSI2GX7fEnXSloh6fKIuGOChx+5xg/ZYP/ljf2XL/Zd3th/+WLf5Y39N72TJb1G0m22N1a3vV3SH0r6O9v7SXpYe8+vn5ojYu6tBAAAAIAUpLxEDQAAAACmQuAAAAAAKEb2gWP7LNt32N5je+xlFm2fZvvrtjfbvrDNbcR4tp9o+zrb36z+PHjM/Xbb3lh9cLGJDi33WrK92vbHq+/fUl3jHomYYP+9zvb/Dr3ezu1iO7Ev25fbvtf2yPd688D7qn37NdvPaXsbMd4E++8U2w8Mvfbe0fY2YjTbR9r+ou07q2POC0bch9dfQrIPHEm3S3qFpJvG3cH2CkkfkPRiScdJerXt49rZPCzjQknXR8Qxkq6vvh7lxxFxfPVxRnubh2ETvpZeL+n+iFgn6W8k/WW7W4lxpvi78ONDr7eZL9OJ2n1E0lLvZfFiScdUH+dJ+mAL24TJfURL7z9J+o+h1967WtgmTGaXpLdGxHGSnivpTSP+7uT1l5DsAyciNkXE15e520mSNkfE3RHxE0lXSTqz+a3DBM6UdEX1+RWSXtbhtmB5k7yWhvfpJyS9oLokJLrH34UZi4ibJP1gibucKemjMXCzpINsH9bO1mE5E+w/JCoitkbEV6rPfyRpk6TDF92N119Csg+cCR0u6btDX9+jff/DRDcOiYit1effl3TImPvtb3uD7ZttE0HdmeS19NP7RMQuSQ9IelIrW4flTPp34W9XSyw+YfvIEd9Hmvi3Ln+/Yvurtj9v++e73hjsq1p2fYKkWxZ9i9dfQjp5H5xp2f6CpENHfOuiiPhs29uD6Sy1/4a/iIiwPe665U+LiC22nyHpBtu3RcRddW8rAP2LpI9FxCO2/0iDadypHW8T0Adf0eDfuh22T5f0GQ2WOyERttdI+qSkN0fEg11vD8bLInAi4oVzPsQWScO/hTyiug0tWGr/2d5m+7CI2FqNcu8d8xhbqj/vtn2jBr89IXDaN8lraeE+91Rv1vUESfe1s3lYxrL7LyKG99Wlkt7TwnahHvxbl7HhA+aIuMb2P9heGxHbu9wuDNheqUHcXBkRnxpxF15/CenLErVbJR1j+yjbqySdLYkrcaVhvaRzqs/PkbTPRM72wbZXV5+v1eAdcO9sbQsxbJLX0vA+faWkG4J3FE7Fsvtv0ZrxMzRYa448rJf02upqTs+V9MDQEmAkzvahC+cr2j5Jg2M0fjmUgGq/XCZpU0S8d8zdeP0lJIsJzlJsv1zS+yU9WdLnbG+MiBfZfoqkSyPi9IjYZft8SddKWiHp8oi4o8PNxl4XS7ra9uslfVvSqyTJg0t+vyEizpX0TEkftr1Hg7/wL44IAqcD415Ltt8laUNErNfgH4F/tL1ZgxNqz+5uizFswv33x7bP0OCqQT+Q9LrONhiPYvtjkk6RtNb2PZLeKWmlJEXEhyRdI+l0SZslPSTp97vZUowywf57paQ32t4l6ceSzuaXQ8k4WdJrJN1me2N129slPVXi9Zci89oBAAAAUIq+LFEDAAAA0AMEDgAAAIBiEDgAAAAAikHgAAAAACgGgQMAAACgGAQOAAAAgGIQOAAAAACKQeAAQI/Z/iXbX7O9v+3H2b7D9rO63i4AAGbFG30CQM/Zfrek/SU9VtI9EfEXHW8SAAAzI3AAoOdsr5J0q6SHJT0vInZ3vEkAAMyMJWoAgCdJWiPpQA0mOQAAZIsJDgD0nO31kq6SdJSkwyLi/I43CQCAme3X9QYAALpj+7WSdkbEP9leIenLtk+NiBu63jYAAGbBBAcAAABAMTgHBwAAAEAxCBwAAAAAxSBwAAAAABSDwAEAAABQDAIHAAAAQDEIHAAAAADFIHAAAAAAFOP/AfsFqK+MpVmMAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qxR8sZ_Y10H7"
      },
      "source": [
        "# Perform Validation on Some Sample Contexts From Research Articles"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5h4BR7VJ10H7",
        "outputId": "45798318-0576-494f-cbec-5ebdd8dad8e7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Context: My feet crunch on gravel as I finish up and slip into my home.  My neighbor’s cursing wakes me up in the morning.I hear damn! and bollocks! I split the blinds with two fingers. The sun shines very bright and I squint to see my other neighbors file outside\n",
            "Question: What was the weather like?\n",
            "Answer: [UNK] sun shines very bright\n",
            "\n",
            "Context: A mom’s job is never truly finished — Elise knew this instinctively when her son was born. For at least the hundredth time, she sat nextto his bed in that chair mending his quilt. There was no way of knowing, the day she bought it at a flea market, that it would become herson’s most valued possession. To call it a quilt stretched the definition as it was nothing more than a thousand pieces of oddly shapedswatches stitched together, layer after layer, until the whole of it was thick enough to hold in the warmth.\n",
            "Question: How many fragments it takes to make a quilt?\n",
            "Answer: a thousand\n",
            "\n",
            "Context: There are still rusted bayonets to be found in the dirt. Alongside broken firearms, canteens, and bullet-struck helmets. At times, stillattached to skeletons. The deep-sea team would occasionally find a corroded tank or the remains of a submarine acting as an aquarium.Fighter planes would turn up far off in the mountains, a surprise to climbers. Rare was it that Hisao found letters buried in Saipan.He dropped his shovel and knelt, the archaeology team at work behind him—industrial lights illuminated the tunnel.\n",
            "Question: What did planes do?\n",
            "Answer: turn up far off in the mountains , a surprise to climbers\n",
            "\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "\n",
        "# To load different check points uncomment next two lines and use the checkpoint directory\n",
        "model = AutoModelForQuestionAnswering.from_pretrained(\"./drive/My Drive/distilbert-aqa-100/aqa-clone\")\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"./drive/My Drive/distilbert-aqa-100/aqa-clone\")\n",
        "\n",
        "contexts = [\"My feet crunch on gravel as I finish up and slip into my home.  My neighbor’s cursing wakes me up in the morning.\"\n",
        "            \"I hear damn! and bollocks! I split the blinds with two fingers. The sun shines very bright and I squint to see my other neighbors file outside\",\n",
        "\n",
        "            \"A mom’s job is never truly finished — Elise knew this instinctively when her son was born. For at least the hundredth time, she sat next\"\n",
        "            \"to his bed in that chair mending his quilt. There was no way of knowing, the day she bought it at a flea market, that it would become her\"\n",
        "            \"son’s most valued possession. To call it a quilt stretched the definition as it was nothing more than a thousand pieces of oddly shaped\"\n",
        "            \"swatches stitched together, layer after layer, until the whole of it was thick enough to hold in the warmth.\",\n",
        "\n",
        "            \"There are still rusted bayonets to be found in the dirt. Alongside broken firearms, canteens, and bullet-struck helmets. At times, still\"\n",
        "            \"attached to skeletons. The deep-sea team would occasionally find a corroded tank or the remains of a submarine acting as an aquarium.\"\n",
        "            \"Fighter planes would turn up far off in the mountains, a surprise to climbers. Rare was it that Hisao found letters buried in Saipan.\"\n",
        "            \"He dropped his shovel and knelt, the archaeology team at work behind him—industrial lights illuminated the tunnel.\"\n",
        "           ]\n",
        "\n",
        "questions = [\n",
        "    \"What was the weather like?\",\n",
        "    \"How many fragments it takes to make a quilt?\",\n",
        "    \"What did planes do?\",\n",
        "]\n",
        "\n",
        "for context, question in zip(contexts,questions):\n",
        "    inputs = tokenizer(question, context, add_special_tokens=True, return_tensors=\"pt\")\n",
        "    input_ids = inputs[\"input_ids\"].tolist()[0]\n",
        "    text_tokens = tokenizer.convert_ids_to_tokens(input_ids)\n",
        "    answer_start_scores, answer_end_scores = model(**inputs)\n",
        "    answer_start = torch.argmax(\n",
        "        answer_start_scores\n",
        "    )  # Get the most likely beginning of answer with the argmax of the score\n",
        "    answer_end = torch.argmax(answer_end_scores) + 1  # Get the most likely end of answer with the argmax of the score\n",
        "    answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(input_ids[answer_start:answer_end]))\n",
        "    print(f\"Context: {context}\")\n",
        "    print(f\"Question: {question}\")\n",
        "    print(f\"Answer: {answer}\",end=\"\\n\\n\")"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "DhHZ7lo43r4x",
        "7LzU3mFTc61u",
        "91EAevf1WigE",
        "VcVQTBCX2Rm9",
        "aEMWxY4q2Yot",
        "9lG7VeaY2J1z",
        "ta_Tyenf2MEi",
        "9rJwSON1YmU4",
        "ESKetF8z_ojh"
      ],
      "machine_shape": "hm",
      "name": "BERT-plotting-loss-backup",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}