notebooks/QQP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import checklist\n",
    "import spacy\n",
    "import itertools\n",
    "\n",
    "import checklist.editor\n",
    "import checklist.text_generation\n",
    "from checklist.test_types import MFT, INV, DIR\n",
    "from checklist.expect import Expect\n",
    "from checklist.test_suite import TestSuite\n",
    "import numpy as np\n",
    "import spacy\n",
    "from checklist.perturb import Perturb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<checklist.text_generation.TextGenerator at 0x7f798146dcc0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor = checklist.editor.Editor()\n",
    "editor.tg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"\tWhat does the Quran say about homosexuality?\t0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "qs = []\n",
    "labels = []\n",
    "all_questions = set()\n",
    "for x in open('/home/marcotcr/datasets/glue/glue_data/QQP/dev.tsv').readlines()[1:]:\n",
    "    try:\n",
    "        q1, q2, label = x.strip().split('\\t')[3:]\n",
    "    except:\n",
    "        print(x)\n",
    "        continue\n",
    "    all_questions.add(q1)\n",
    "    all_questions.add(q2)\n",
    "    qs.append((q1, q2))\n",
    "    labels.append(label)\n",
    "labels = np.array(labels).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_questions = list(all_questions)\n",
    "parsed_questions = list(nlp.pipe(all_questions))\n",
    "spacy_map = dict([(x, y) for x, y in zip(all_questions, parsed_questions)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if you want to pickle this for later use:\n",
    "# pickle.dump(spacy_map, open(path, 'wb'))\n",
    "# Then run the following instead of the cell above whenver you run the notebook\n",
    "# spacy_map2 =  pickle.load(open(path, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "parsed_qs = [(spacy_map[q[0]], spacy_map[q[1]]) for q in qs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "suite = TestSuite()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "journalist, historian, nurse, engineer, accountant, attorney, architect, artist, editor, actor, actress, analyst, intern, economist, entrepreneur, author, assistant, interpreter, investor, executive, organizer, escort, educator, agent, academic, activist, advisor, administrator, investigator, auditor\n"
     ]
    }
   ],
   "source": [
    "professions = editor.suggest('{first_name} works as {a:mask}.')[:30]\n",
    "print(', '.join(professions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "professions = editor.suggest('{first_name} works as {a:mask}.')[:30]\n",
    "professions += editor.suggest('{first_name} {last_name} works as {a:mask}.')[:30]\n",
    "professions = list(set(professions))\n",
    "# professions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "guy, player, writer, actor, example, man, person, friend, kid, hitter, read, shooter, one, coach, dude, pick, teacher, shot, fighter, artist, student, quarterback, poet, character, reader, sport, name, bet, cook, reporter\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('{first_name} {last_name} is a good {mask}.')[:30]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_nouns = ['player', 'person', 'friend', 'kid', 'candidate']\n",
    "nouns = list(set(professions + other_nouns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "effective, actual, important, American, active, successful, good, independent, outstanding, experienced, excellent, elite, honest, influential, real, interested, Australian, international, appropriate, inspiring, better, established, accomplished, serious, great, aspiring, true, outside, ethical, English, bad, credible, LGBT, acceptable, average, interesting, exceptional, Israeli, incompetent, unusual, official, expert, ordinary, adequate, amazing, Irish, political, amateur, accurate, career\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Is {first_name} {last_name} {a:mask} {noun}?', noun=nouns)[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "adjs = ['effective', 'actual', 'American', 'active', 'honest', 'excellent', 'elite', 'acomplished', 'official', 'outstanding', 'experienced', 'independent', 'international', 'aspiring', 'average', 'good', 'amazing', 'exceptional', 'successful', 'accredited', 'English', 'real', 'bad', 'terrible', 'fake', 'unusual', 'influential', 'incompetent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(('Is {first_name} {last_name} {a:noun}?', 'Is {first_name} {last_name} {a:adj} {noun}?'),\n",
    "                noun=nouns,\n",
    "                adj=adjs,\n",
    "                remove_duplicates=True, \n",
    "                nsamples=1000)\n",
    "test = MFT(**t, labels=0, name='Modifier: adj', capability='Vocabulary', \n",
    "          description = 'Adding an adjective makes questions non-duplicates')\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Is person {a1, a2}?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dead, gay, alive, right, Dead, real, Jewish, mad, wrong, correct, insane, back, crazy, related, OK, Right, immortal, evil, straight, interested, famous, racist, sane, next, President, alone, lying, relevant, okay, DEAD, bisexual, God, angry, guilty, ok, Alone, Wrong, ready, gone, innocent, King, human, Muslim, Black, assassinated, Batman, here, Real, dying, involved, homosexual, special, Famous, happy, free, murdered, Superman, president, joking, White, Back, Gay, Catholic, cool, safe, resurrected, killed, American, missing, corrupt, cursed, alright, single, Crazy, Mad, Satan, Missing, legit, shot, Evil, finished, worthy, doomed, sincere, aware, autistic, responsible, dangerous, Christian, Cool, Immortal, radioactive, remembered, Next, serious, cheating, different, popular, Legend, listening, now, illegitimate, homophobic, kidding, sick, smart, connected, trustworthy, Innocent, legal\n",
      "\n",
      "racist, atheist, outlaw, asshole, actor, ape, American, inspiration, idiot, orphan, Christian, ancestor, artist, Avenger, anomaly, icon, anarchist, homosexual, influence, alien, inventor, author, Alien, android, immortal, Immortal, idol, Indian, angel, assassin, African, Arab, intellectual, adult, extremist, Assassin, imperialist, alcoholic, Icon, rapist, example, Italian, Australian, archetype, Armenian, exception, murderer, activist, astronaut, expert\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Is John Wayne {mask}?')))\n",
    "print()\n",
    "print(', '.join(editor.suggest('Is John Wayne {a:mask}?')[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "adjs_without_overlap = ['dead', 'gay', 'Jewish', 'Christian', 'American', 'mad', 'immortal', 'evil', 'famous', 'racist', 'Muslim', 'white', 'black', 'English', 'autistic', 'Australian', 'trustworthy', 'an atheist', 'an anarchist', 'an inventor', 'Indian', 'Armenian', 'an astronaut', 'an immigrant']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name} {last_name} {adj1}?',\n",
    "    'Is {first_name} {last_name} {adj2}?',\n",
    "    ),\n",
    "    adj=adjs_without_overlap,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "test = MFT(**t, labels=0, name='different adjectives', capability = 'Vocabulary',\n",
    "          description='Same first and last name, different adjectives')\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "different animals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat, dog, rabbit, turtle, too, spider, now, rat, name, goat, lizard, dragon, named, pig, wolf, monkey, girl, squirrel, owl, also, here, called, there, tiger, python, friend, lab, phone, mix, bear, bird, …, boy, farm, duck, snake, zoo, owner, problem, fish, carrier, Shiva, shop, myself, companion, boyfriend, kitten, store, dinosaur, animal, watch, ring, project, deer, Labrador, Shepherd, tree, killer, lobster, mom, dish, X, bunny, girlfriend, shark, already, gun, car, puppy, one, computer, elephant, fetish, bug, brother, lover, tattoo, monitor, snail, cow, XD, Goat, mouse, somewhere, robot, sometimes, mother, syndrome, haha, pair, knife, life, carriage, Persian, cats, collar, that, diary, door, family, boo, number, show, assistant, toy, horse, chicken, laptop, server, tracker\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('I have a pet {mask}.')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "animals = ['cat', 'dog', 'rabbit', 'turtle', 'spider', 'rat', 'goat', 'lizard', 'pig', 'monkey', 'squirrel', 'owl', 'snake', 'fish', 'lobster', 'snail', 'chicken']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "food, pellets, eggs, water, again, worms, properly, now, meat, today, too, anything, outside, here, something, tonight, poop, fish, this, milk, directly, that, myself, rice, instead, nuts, there, live, steak, more, dinner, formula, treats, poison, regularly, correctly, rabbit, right, blood, enough, back, chicken, cheese, once, larvae, peanuts, feed, alive, honey, grass, feces, soup, anyway, scraps, bugs, what, alone, breakfast, oil, raw, well, chocolate, urine, corn, cereal, sugar, seeds, home, liver, free, better, protein, online, daily, cookies, carrots, lunch, yogurt, candy, cat, grain, seed, spinach, anymore, salad, fat, antibiotics, greens, Rice, it, indoors, …, shit, inside, money, toys, crap, organic, naturally, hay, bacon, some, cats, bird, bamboo, egg, breast, separately, rats, juice, vegetables, biscuits, waste, butter, crow, insulin, soy, sometimes, meal, mix, bones, yet, fly, appropriately, balls, privately, bananas, tuna, salmon, chickens, Milo, birds, bread, saliva, chicks, beef, tea, fruit, snakes, banana, salt, tail, broccoli, normally, wild, sushi, friends, leaves, trout, shrimp, tomorrow, chips, one, snake, rabbits, twice, grains, yourself, gluten, tails, safely, mate, differently, white, life, baby, mine, DNA, shells, shell, snail, Milk, venom, owl, breasts, poisonous, bites, mail, plants, bait, pork, wings, kale, kittens, paste, oats, traps, GMOs, healthy, beans, yesterday, slime, fries, though, chick, commercially, tomatoes, fingers, locally, cake, iron, vitamins, ok, python, grapes, oils, marijuana, strawberries, stew, GMO, soap, supplements, ducks, frogs, crabs, occasionally, alcohol, lettuce, babies, stock, responsibly, pudding, bills, tacos, pigs, elsewhere, pancakes, love, illegally, young, poultry, dry, please, kidney, first, outdoors, capsules, turkey, feathers, junk, jelly, snacks, bug, legally, scratch, how, straight, somehow, variety, toy, exclusively, these, feather, weed, green, samples, rat, habit, popcorn, also, black, bill, litter, friend, pod, beetles, away, externally, flour, manure, kidneys, hair, pack, feet, any, parasites, trap, wine, afterwards, stuff, ass, dog, freely, badly, fur, garbage\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Can I feed my {an} {mask}?', an=animals)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "food = ['eggs', 'water',  'worms', 'meat', 'poop', 'milk', 'rice', 'nuts', 'steak', 'formula',  'soup', 'bugs', 'oil', 'chocolate', 'corn', 'cereal', 'sugar', 'seeds', 'liver', 'cookies', 'carrots', 'yogurt', 'salad', 'greens', 'rice', 'bananas', 'tuna', 'apples', 'salmon', 'butter', 'insulin', 'soy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Can I feed my {animal1} {food}?',\n",
    "    'Can I feed my {animal2} {food}?',\n",
    "    ),\n",
    "    animal=animals,\n",
    "    food=food,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'Different animals' \n",
    "desc = 'Ask the same question about two different pet animals, expect prediction to be 0'\n",
    "test = MFT(**t, labels=0, name=name, capability = 'Vocabulary',\n",
    "          description=desc)\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=5)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Modifiers that don't matter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "action = editor.suggest('Is that {animal} really {mask} on the couch?', animal=animals)[:30]\n",
    "\n",
    "editor.suggest('Is that {animal} {mask} {action} on the couch?', animal=animals, action=action)\n",
    "non_changing_modifier = ['really', 'truly', 'actually', 'indeed', 'in fact', 'currently', 'literally', 'somehow']\n",
    "t = editor.template((\n",
    "    'Is that {animal} {action} on the {place}?',\n",
    "    'Is that {animal} {mod2} {action} on the {place}?',\n",
    "    ),\n",
    "    action=action,\n",
    "    animal=animals,\n",
    "    mod=non_changing_modifier,\n",
    "    place =['couch','bed', 'sofa', 'table'],\n",
    "    remove_duplicates=False, \n",
    "    nsamples=1000)\n",
    "name = 'Irrelevant modifiers - animals' \n",
    "desc = 'Add modifiers that preserve question semantics (e.g. \\'really\\')'\n",
    "test = MFT(**t, labels=1, name=name, capability = 'Vocabulary',\n",
    "          description=desc)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "action = editor.suggest('Is {first_name1} {mask} to {first_name2}?')[:30]\n",
    "editor.suggest('Is {first_name1} {mask} {a} to {first_name2}?', a=action)\n",
    "non_changing_modifier = ['really', 'truly', 'actually', 'indeed', 'in fact']\n",
    "t = editor.template((\n",
    "    'Is {first_name1} {action} to {first_name2}?',\n",
    "    'Is {first_name1} {mod2} {action} to {first_name2}?',\n",
    "    ),\n",
    "    action=action,\n",
    "    mod=non_changing_modifier,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'Irrelevant modifiers - people' \n",
    "desc = 'Add modifiers that preserve question semantics (e.g. \\'really\\')'\n",
    "test = MFT(**t, labels=1, name=name, capability = 'Vocabulary',\n",
    "          description=desc)\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'My pet {animal1} eats {food}. Is it normal for animals to eat {food}?',\n",
    "    'My pet {animal2} eats {food}. Is it normal for animals to eat {food}?',\n",
    "    ),\n",
    "    animal=animals,\n",
    "    food=food,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'Irrelevant preamble with different examples.' \n",
    "desc = 'Ask questions about animals in general, but with different examples in the preamble. Expect duplicate.'\n",
    "test = MFT(**t, labels=1, name=name, capability = 'Vocabulary',\n",
    "          description=desc)\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6cf78f9d78c42d0b462db9bc203c965",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "TemplateEditor(bert_suggests=['arm', 'leg', 'wrist', 'back', 'hand', 'neck', 'foot', 'ankle', 'nose', 'thumb',…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "editor.visual_suggest('I broke my {mask} playing tennis.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nouns = editor.selected_suggestions\n",
    "nouns = ['finger', 'forearm', 'feet', 'head', 'foot', 'elbow', 'nose', 'back', 'heart', 'shoulder', 'skull', 'toe', 'hip', 'neck', 'body', 'thumb', 'rib', 'knee', 'jaw', 'heel', 'thigh', 'ankle', 'arm', 'wrist', 'eye', 'spine', 'leg', 'butt', 'teeth', 'skin', 'bone', 'hand', 'face']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'I hurt my {n} last time I played {sport}. {followup}',\n",
    "    'I hurt my {n2} last time I played {sport}. {followup}',\n",
    "    ),\n",
    "    n=nouns,\n",
    "    sport=['tennis', 'golf', 'soccer', 'football'],\n",
    "    followup=['Is this going to impact my performance?',\n",
    "              'Should I never play again?',\n",
    "              'Is this a common injury?',\n",
    "             'Is it normal to hurt this part of the body?',],\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'Preamble is relevant (different injuries)' \n",
    "desc = 'Question preamble mentions different injuries, which makes the questions themselves non-duplicates.'\n",
    "test = MFT(**t, labels=0, name=name, capability = 'Vocabulary',\n",
    "          description=desc)\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Taxonomy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Synonyms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('effective', 'efficient'),\n",
      "('hateful', 'mean'),\n",
      "('critical', 'decisive', 'vital'),\n",
      "('worried', 'upset'),\n",
      "('important', 'significant', 'authoritative'),\n",
      "('resilient', 'live'),\n",
      "('consistent', 'uniform', 'logical', 'coherent'),\n",
      "('enlightened', 'educated', 'clear'),\n",
      "('corrupt', 'corrupted'),\n",
      "('radical', 'revolutionary'),\n",
      "('addicted', 'addict'),\n",
      "('suspicious', 'suspect', 'wary'),\n",
      "('insecure', 'unsafe'),\n",
      "('controlling', 'moderate', 'control'),\n",
      "('positive', 'confident'),\n",
      "('open', 'clear', 'capable', 'receptive', 'candid'),\n",
      "('professional', 'pro'),\n",
      "('activist', 'militant'),\n",
      "('evil', 'vicious'),\n",
      "('independent', 'autonomous'),\n",
      "('anxious', 'nervous'),\n",
      "('charitable', 'sympathetic', 'benevolent'),\n",
      "('spiritual', 'religious'),\n",
      "('confident', 'positive'),\n",
      "('kind', 'tolerant'),\n",
      "('scared', 'frightened'),\n",
      "('sensitive', 'sensible'),\n",
      "('so', 'then'),\n",
      "('clear', 'open', 'clean', 'light', 'clearly'),\n",
      "('specific', 'particular'),\n",
      "('individual', 'single', 'private', 'person'),\n",
      "('passive', 'inactive', 'peaceful'),\n",
      "('desperate', 'heroic'),\n",
      "('dependent', 'qualified'),\n",
      "('conservative', 'cautious'),\n",
      "('frustrated', 'disappointed', 'queer', 'defeated'),\n",
      "('rigid', 'strict', 'fixed', 'stiff'),\n",
      "('vocal', 'outspoken'),\n",
      "('competitive', 'militant'),\n",
      "('nervous', 'anxious'),\n",
      "('progressive', 'liberal', 'imperfect'),\n",
      "('extremist', 'radical'),\n",
      "('cautious', 'conservative', 'timid'),\n",
      "('honest', 'true', 'good', 'reliable', 'honorable', 'fair'),\n",
      "('depressed', 'blue'),\n",
      "('upset', 'worried', 'confused', 'broken', 'disturbed', 'distressed'),\n",
      "('aware', 'mindful'),\n",
      "('courageous', 'brave'),\n",
      "('organised', 'organized', 'direct', 'engineer'),\n",
      "('committed', 'attached'),\n",
      "('ambitious', 'challenging'),\n",
      "('knowledgeable', 'learned', 'intimate'),\n",
      "('alone', 'solitary', 'lonely'),\n",
      "('healthy', 'intelligent', 'sound', 'respectable'),\n",
      "('hungry', 'thirsty'),\n",
      "('authentic', 'reliable'),\n",
      "('bad', 'spoiled', 'sorry', 'risky', 'tough', 'defective'),\n",
      "('unhappy', 'distressed'),\n",
      "('disconnected', 'confused', 'fragmented'),\n",
      "('fit', 'set'),\n",
      "('fearful', 'terrible', 'awful', 'cowardly'),\n",
      "('hopeful', 'promising', 'bright'),\n",
      "('biased', 'bias'),\n",
      "('lonely', 'alone', 'solitary'),\n",
      "('religious', 'spiritual'),\n",
      "('mindful', 'aware'),\n",
      "('stressed', 'stress'),\n",
      "('fat', 'productive', 'rich', 'fatty'),\n",
      "('disruptive', 'troubled'),\n",
      "('smart', 'wise', 'chic', 'bright'),\n",
      "('mean', 'hateful', 'average'),\n",
      "('thoughtful', 'attentive'),\n",
      "('needy', 'impoverished'),\n",
      "('rude', 'primitive', 'crude'),\n",
      "('demanding', 'exact'),\n",
      "('emotional', 'excited'),\n",
      "('understanding', 'savvy'),\n",
      "('efficient', 'effective'),\n",
      "('alienated', 'alien', 'estranged'),\n",
      "('educated', 'enlightened'),\n",
      "('free', 'liberal', 'innocent'),\n",
      "('tolerant', 'resistant', 'liberal', 'kind'),\n",
      "('militant', 'activist', 'competitive'),\n",
      "('capable', 'able', 'open'),\n",
      "('intelligent', 'healthy', 'sound', 'thinking'),\n",
      "('active', 'alive', 'dynamic'),\n",
      "('organized', 'organised', 'direct'),\n",
      "('grateful', 'thankful'),\n",
      "('ethical', 'honorable'),\n",
      "('strict', 'rigid', 'stern'),\n",
      "('innovative', 'modern', 'advanced'),\n",
      "('authoritarian', 'dictator'),\n",
      "('humble', 'modest', 'small'),\n",
      "('inspired', 'divine'),\n",
      "('shy', 'timid', 'unsure')\n"
     ]
    }
   ],
   "source": [
    "tmp = []\n",
    "x = editor.suggest('How can I become more {mask}?')\n",
    "x += editor.suggest('How can I become less {mask}?')\n",
    "for a in set(x):\n",
    "    e = editor.synonyms('How can I become {moreless} %s?' % a, a, moreless=['more', 'less'])\n",
    "    if e:\n",
    "#         print(a, [b[0][0] for b in e] )\n",
    "        tmp.append([a] + e)\n",
    "#         opps.append((a, e[0][0][0]))\n",
    "print(',\\n'.join([str(tuple(x)) for x in tmp]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "synonyms = [ ('spiritual', 'religious'), ('angry', 'furious'), ('organized', 'organised'),\n",
    "            ('vocal', 'outspoken'), ('grateful', 'thankful'), ('intelligent', 'smart'),\n",
    "            ('humble', 'modest'), ('courageous', 'brave'), ('happy', 'joyful'), ('scared', 'frightened'),\n",
    "           ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Antonyms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('impatient', 'patient'),('insecure', 'secure'),('positive', 'negative', 'neutral'),('organic', 'functional'),('stupid', 'intelligent', 'smart'),('powerless', 'powerful'),('defensive', 'offensive'),('specific', 'general'),('individual', 'common'),('passive', 'active'),('dependent', 'independent'),('conservative', 'progressive', 'liberal'),('conspicuous', 'invisible'),('pessimistic', 'optimistic'),('negative', 'positive'),('progressive', 'conservative'),('cautious', 'brave'),('courageous', 'fearful'),('visible', 'invisible'),('capitalist', 'socialist'),('hungry', 'thirsty'),('bad', 'good'),('unhappy', 'happy'),('irresponsible', 'responsible'),('hopeful', 'hopeless'),('religious', 'secular'),('fat', 'lean', 'thin'),('smart', 'stupid'),('rude', 'civil', 'polite'),('emotional', 'intellectual'),('active', 'passive'),('optimistic', 'pessimistic'),('humble', 'proud'),('shy', 'confident')\n"
     ]
    }
   ],
   "source": [
    "opps = []\n",
    "x = editor.suggest('How can I become more {mask}?')\n",
    "x += editor.suggest('How can I become less {mask}?')\n",
    "for a in set(x):\n",
    "    e = editor.antonyms('How can I become {moreless} %s?' % a, a, moreless=['more', 'less'])\n",
    "    if e:\n",
    "#         print(a, [b[0][0] for b in e] )\n",
    "        opps.append([a] + e)\n",
    "#         opps.append((a, e[0][0][0]))\n",
    "print(','.join([str(tuple(x)) for x in opps]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "antonyms = [('progressive', 'conservative'),('religious', 'secular'),('positive', 'negative'),('defensive', 'offensive'),('rude',  'polite'),('optimistic', 'pessimistic'),('stupid', 'smart'),('negative', 'positive'),('unhappy', 'happy'),('active', 'passive'),('impatient', 'patient'),('powerless', 'powerful'),('visible', 'invisible'),('fat', 'thin'),('bad', 'good'),('cautious', 'brave'), ('hopeful', 'hopeless'),('insecure', 'secure'),('humble', 'proud'),('passive', 'active'),('dependent', 'independent'),('pessimistic', 'optimistic'),('irresponsible', 'responsible'),('courageous', 'fearful')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "t = editor.template([\n",
    "    (\n",
    "    'How can I become more {x[0]}?',\n",
    "    'How can I become more {x[1]}?',\n",
    "    ),\n",
    "    (\n",
    "    'How can I become more {x[1]}?',\n",
    "    'How can I become more {x[0]}?',\n",
    "    ),\n",
    "    (\n",
    "    'How can I become less {x[0]}?',\n",
    "    'How can I become less {x[1]}?',\n",
    "    ),\n",
    "    (\n",
    "    'How can I become less {x[1]}?',\n",
    "    'How can I become less {x[0]}?',\n",
    "    ),\n",
    "    (\n",
    "    'How can I become {a:x[0]} person?',\n",
    "    'How can I become {a:x[1]} person?',\n",
    "    ),\n",
    "    (\n",
    "    'How can I become {a:x[1]} person?',\n",
    "    'How can I become {a:x[0]} person?',\n",
    "    ),\n",
    "],\n",
    "    unroll=True,\n",
    "    x=synonyms,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'How can I become more {synonym}?' \n",
    "desc = 'different (simple) templates where words are replaced with their synonyms'\n",
    "test = MFT(**t, labels=1, name=name, capability = 'Taxonomy',\n",
    "          description=desc)\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import re\n",
    "def replace_pairs(pairs):\n",
    "    def replace_z(text):\n",
    "        ret = []\n",
    "        for x, y in pairs:\n",
    "            t = re.sub(r'\\b%s\\b' % x, y, text )\n",
    "            if t != text:\n",
    "                ret.append(t)\n",
    "            if y == 'smart':\n",
    "                continue\n",
    "            t = re.sub(r'\\b%s\\b' % y, x, text )\n",
    "            if t != text:\n",
    "                ret.append(t)\n",
    "        return list(set(ret))\n",
    "    return replace_z\n",
    "def apply_and_pair(fn):\n",
    "    def ret_fn(text):\n",
    "        ret = fn(text)\n",
    "        return [(text, r) for r in ret]\n",
    "    return ret_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('spiritual', 'religious'),\n",
       " ('angry', 'furious'),\n",
       " ('organized', 'organised'),\n",
       " ('vocal', 'outspoken'),\n",
       " ('grateful', 'thankful'),\n",
       " ('intelligent', 'smart'),\n",
       " ('humble', 'modest'),\n",
       " ('courageous', 'brave'),\n",
       " ('happy', 'joyful'),\n",
       " ('scared', 'frightened')]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "synonyms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = '(question, f(question)) where f(question) replaces synonyms?' \n",
    "desc = 'Expect 1, should be easy because it\\'s individual word changes'\n",
    "t = Perturb.perturb(list(all_questions), apply_and_pair(replace_pairs(synonyms)), nsamples=1000, keep_original=False)\n",
    "test = INV(t.data, threshold=0.1, name=name, description=desc, capability='Taxonomy')\n",
    "# test.run(new_pp, n=500, seed=1)\n",
    "# test.summary(n=3)\n",
    "suite.add(test, overwrite=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_to_each_and_product(fn):\n",
    "    def apply_to_one(x):\n",
    "        p = fn(x)\n",
    "        if not p:\n",
    "            p = []\n",
    "        return list(set([x] + p))\n",
    "    def ret_fn(pair):\n",
    "        p1 = apply_to_one(pair[0])\n",
    "        p2 = apply_to_one(pair[1])\n",
    "        return [x for x in itertools.product(p1, p2) if x != pair]\n",
    "    return ret_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = 'Replace synonyms in real pairs'\n",
    "desc = ''\n",
    "t = Perturb.perturb(qs, apply_to_each_and_product(replace_pairs(synonyms)), nsamples=1000, keep_original=True)\n",
    "test = INV(t.data, threshold=0.1, name=name, description=desc, capability='Taxonomy')\n",
    "# test.run(new_pp, n=500, seed=1)\n",
    "# test.summary(n=3)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "t = editor.template([(\n",
    "    'How can I become more {x[0]}?',\n",
    "    'How can I become less {x[0]}?',\n",
    "    ),\n",
    "    (\n",
    "    'How can I become less {x[1]}?',\n",
    "    'How can I become more {x[1]}?',\n",
    "    )],\n",
    "    unroll=True,\n",
    "    x=antonyms,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'How can I become more X != How can I become less X' \n",
    "desc = ''\n",
    "test = MFT(**t, labels=0, name=name, capability = 'Vocabulary',\n",
    "          description=desc)\n",
    "# test.run(new_pp, n=500, seed=1)\n",
    "# test.summary(n=3)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "t = editor.template([(\n",
    "    \n",
    "    'How can I become more {x[0]}?',\n",
    "    'How can I become less {x[1]}?',\n",
    "    ),\n",
    "    (\n",
    "    'How can I become less {x[0]}?',\n",
    "    'How can I become more {x[1]}?',\n",
    "    )],\n",
    "    unroll=True,\n",
    "    x=antonyms,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'How can I become more X = How can I become less antonym(X)' \n",
    "desc = ''\n",
    "test = MFT(**t, labels=1, name=name, capability = 'Taxonomy',\n",
    "          description=desc)\n",
    "# test.run(new_pp, n=500, seed=1)\n",
    "# test.summary(n=3)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DIR version (kinda bad, won't add to suite)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(list(all_questions), apply_and_pair(replace_pairs(antonyms)), nsamples=200, keep_original=False)\n",
    "test = DIR(t.data, expect=Expect.eq(0), agg_fn='all')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=5)\n",
    "# suite.add(test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wrap_apply_to_each(fn, both=False, *args, **kwargs):\n",
    "    def new_fn(qs, *args, **kwargs):\n",
    "        q1, q2 = qs\n",
    "        ret = []\n",
    "        fnq1 = fn(q1, *args, **kwargs)\n",
    "        fnq2 = fn(q2, *args, **kwargs)\n",
    "        if type(fnq1) != list:\n",
    "            fnq1 = [fnq1]\n",
    "        if type(fnq2) != list:\n",
    "            fnq2 = [fnq2]\n",
    "        ret.extend([(x, str(q2)) for x in fnq1])\n",
    "        ret.extend([(str(q1), x) for x in fnq2])\n",
    "        if both:\n",
    "            ret.extend([(x, x2) for x, x2 in itertools.product(fnq1, fnq2)])\n",
    "        return [x for x in ret if x[0] and x[1]]\n",
    "    return new_fn\n",
    "def wrap_apply_to_both(fn, *args, **kwargs):\n",
    "    def new_fn(qs, *args, **kwargs):\n",
    "        q1, q2 = qs\n",
    "        ret = []\n",
    "        fnq1 = fn(q1, *args, **kwargs)\n",
    "        fnq2 = fn(q2, *args, **kwargs)\n",
    "        if type(fnq1) != list:\n",
    "            fnq1 = [fnq1]\n",
    "        if type(fnq2) != list:\n",
    "            fnq2 = [fnq2]\n",
    "        ret.extend([(x, x2) for x, x2 in itertools.product(fnq1, fnq2)])\n",
    "        return [x for x in ret if x[0] and x[1]]\n",
    "    return new_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Typos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(qs, wrap_apply_to_each(Perturb.add_typos), nsamples=500)\n",
    "test = INV(t.data, name='add one typo', capability='Robustness', description='')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Contractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(qs, wrap_apply_to_each(Perturb.contractions, both=True), nsamples=500)\n",
    "test = INV(**t, name='contrations', capability='Robustness', description='')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Paraphrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "def me_to_you(text):\n",
    "    t = re.sub(r'\\bI\\b', 'you', text)\n",
    "    t = re.sub(r'\\bmy\\b', 'your', t)\n",
    "    return re.sub(r'\\bmine\\b', 'yours', t)\n",
    "def paraphrases(text):\n",
    "    ts = ['How do I ', 'How can I ', 'What is a good way to ', 'How should I ']\n",
    "    templates1 = ['How do I {x}?', 'How can I {x}?', 'What is a good way to {x}?', 'If I want to {x}, what should I do?',\n",
    "                'In order to {x}, what should I do?']\n",
    "    ts2 = ['Can you ', 'Can I ']#, 'Do I']\n",
    "    ts3 = ['Do I ']\n",
    "    templates2 = ['Can you {x}?', 'Can I {x}?', 'Do you think I can {x}?', 'Do you think you can {x}?',]\n",
    "    templates3 = ['Do I {x}?', 'Do you think I {x}?']\n",
    "    ret = []\n",
    "    for i, (tsz, templates) in enumerate(zip([ts, ts2, ts3], [templates1, templates2, templates3])):\n",
    "        for t in tsz:\n",
    "            if text.startswith(t):\n",
    "                x = text[len(t):].strip('?')\n",
    "                ts = editor.template(templates, x=x).data[0]\n",
    "                if i <= 1:\n",
    "                    ts = ts + [me_to_you(x) for x in ts]\n",
    "                ret += ts\n",
    "    return ret\n",
    "def paraphrases_product(text):\n",
    "    pr = paraphrases(text)\n",
    "    return list(itertools.product(pr, pr))\n",
    "\n",
    "def paraphrase_each(pair):\n",
    "    p1 = paraphrases(pair[0])\n",
    "    p2 = paraphrases(pair[1])\n",
    "    return list(itertools.product(p1, p2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(list(all_questions), paraphrases_product, nsamples=200, keep_original=False)\n",
    "name = '(q, paraphrase(q))'\n",
    "desc = 'For questions that start with \"How do I X\", \"How can I X\", etc'\n",
    "test = DIR(t.data, expect=Expect.eq(1), agg_fn='all', name=name, description=desc, capability='Robustness')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=5)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(qs, paraphrase_each, nsamples=100, keep_original=True)\n",
    "name = 'Product of paraphrases(q1) * paraphrases(q2)'\n",
    "desc = 'For questions that start with \"How do I X\", \"How can I X\", etc'\n",
    "test = INV(t.data, name=name, description=desc, capability='Robustness')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=5)\n",
    "suite.add(test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Change same name, number, location in both"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "person1 and person2 are different by first and last name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name1} {last_name1} {adj}?',\n",
    "    'Is {first_name2} {last_name2} {adj}?',\n",
    "    ),\n",
    "    adj=adjs_without_overlap,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "test = MFT(**t, labels=0, name='same adjectives, different people', capability = 'NER',\n",
    "          description='Different first and last name, same adjectives')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=5)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "person1 and person2 are different by first name only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name} {last_name} {adj}?',\n",
    "    'Is {first_name2} {last_name} {adj}?',\n",
    "    ),\n",
    "    adj=adjs_without_overlap,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "test = MFT(**t, labels=0, name='same adjectives, different people v2', capability = 'NER',\n",
    "          description='Different first name, same adjective and last name')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=5)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "person1 and person2 are different by last name only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name} {last_name} {adj}?',\n",
    "    'Is {first_name} {last_name2} {adj}?',\n",
    "    ),\n",
    "    adj=adjs_without_overlap,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "test = MFT(**t, labels=0, name='same adjectives, different people v3', capability = 'NER',\n",
    "          description='Different last name, same adjective and first name')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=5)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_both_wrapper(fn):\n",
    "    def change_both(qs):\n",
    "        q1, q2 = qs\n",
    "        seed = np.random.randint(100)\n",
    "        c1 = fn(q1, seed=seed, meta=True)\n",
    "        c2 = fn(q2, seed=seed, meta=True)\n",
    "        if not c1 or not c2:\n",
    "            return\n",
    "        c1, m1 = c1\n",
    "        c2, m2 = c2\n",
    "        return [(q1, q2) for q1, q2, m1, m2 in zip(c1, c2, m1, m2) if m1 == m2]\n",
    "    return change_both\n",
    "\n",
    "def change_each_wrapper(fn):\n",
    "    def change_one(qs, **kwargs):\n",
    "        q1, q2 = qs\n",
    "        seed = np.random.randint(100)\n",
    "        c1 = fn(q1, seed=seed, meta=True, **kwargs)\n",
    "        c2 = fn(q2, seed=seed, meta=True, **kwargs)\n",
    "        if not c1 or not c2:\n",
    "            return\n",
    "        c1, m1 = c1\n",
    "        c2, m2 = c2\n",
    "        ret = []\n",
    "        ret.extend([(q1_, str(q2)) for q1_, m1_ in zip(c1, m1) if m1_[0] in str(q2)])\n",
    "        ret.extend([(str(q1), q2_) for q2_, m2_ in zip(c2, m2) if m2_[0] in str(q1)])\n",
    "        return ret\n",
    "    return change_one\n",
    "    \n",
    "t = Perturb.perturb(parsed_qs, change_both_wrapper(Perturb.change_names), nsamples=500)\n",
    "test = INV(**t, name='Change same name in both questions', capability='NER',\n",
    "          description='')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Locs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_qs, change_both_wrapper(Perturb.change_location), nsamples=500)\n",
    "test = INV(**t, name='Change same location in both questions', capability='NER',\n",
    "          description='')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_qs, change_both_wrapper(Perturb.change_number), nsamples=500)\n",
    "test = INV(**t, name='Change same number in both questions', capability='NER',\n",
    "          description='')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Change name, loc, number in only one where orig prediction is duplicate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Changing only first names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# t = Perturb.perturb(parsed_qs, wrap_apply_to_each(Perturb.change_names), nsamples=1500, first_only=True)\n",
    "t = Perturb.perturb(parsed_qs, change_each_wrapper(Perturb.change_names), nsamples=500, first_only=True)\n",
    "expect_fn = Expect.eq(0)\n",
    "expect_fn = Expect.slice_orig(expect_fn, lambda orig, *args: orig == 1)\n",
    "name = 'Change first name in one of the questions'\n",
    "desc = 'Take pairs that are originally predicted as duplicates, change first name in one of them and expect new prediction to be non-duplicate'\n",
    "test = DIR(**t, expect=expect_fn, name=name, description=desc, capability='NER')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Changing first and last names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# t = Perturb.perturb(parsed_qs, wrap_apply_to_each(Perturb.change_names), nsamples=1500)\n",
    "t = Perturb.perturb(parsed_qs, change_each_wrapper(Perturb.change_names), nsamples=1500)\n",
    "name = 'Change first and last name in one of the questions'\n",
    "desc = 'Take pairs that are originally predicted as duplicates, change first and last name in one of them and expect new prediction to be non-duplicate'\n",
    "test = DIR(**t, expect=expect_fn, name=name, description=desc, capability='NER')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Locs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# t = Perturb.perturb(parsed_qs, wrap_apply_to_each(Perturb.change_location), nsamples=1500)\n",
    "t = Perturb.perturb(parsed_qs, change_each_wrapper(Perturb.change_location), nsamples=1500)\n",
    "name = 'Change location in one of the questions'\n",
    "desc = 'Take pairs that are originally predicted as duplicates, change location in one of them and expect new prediction to be non-duplicate'\n",
    "test = DIR(**t, expect=expect_fn, name=name, description=desc, capability='NER')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# t = Perturb.perturb(parsed_qs, wrap_apply_to_each(Perturb.change_number), nsamples=1500)\n",
    "t = Perturb.perturb(parsed_qs, change_each_wrapper(Perturb.change_number), nsamples=1500)\n",
    "name = 'Change numbers in one of the questions'\n",
    "desc = 'Take pairs that are originally predicted as duplicates, change number in one of them and expect new prediction to be non-duplicate'\n",
    "test = DIR(**t, expect=expect_fn, name=name, description=desc, capability='NER')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Change "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keep entities, fill in with BERT gibberish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mask_gibberish(question):\n",
    "    ents = question.ents\n",
    "    if not ents:\n",
    "        return None\n",
    "    wp = [x.text for x in question if x.tag_ in ['WP', 'WRB', 'WDT']]\n",
    "    if not wp:\n",
    "        wp = question[0].text\n",
    "    else:\n",
    "        wp = wp[0]\n",
    "    ents = [x.text for x in ents]\n",
    "    ents[-1] = ents[-1] + '?'\n",
    "    template = ' {mask} '.join([wp] + ents)\n",
    "    gibberish = editor.template(template).data[:5]\n",
    "#     return gibberish\n",
    "    ret = [(question.text, x) for x in gibberish if question.text.lower() != x.lower() ]\n",
    "    return ret\n",
    "def gibberish_both(qs):\n",
    "    q1, q2 = qs\n",
    "    ret = []\n",
    "    x1 = mask_gibberish(q1)\n",
    "    if x1:\n",
    "        ret.extend(x1)\n",
    "    x2 = mask_gibberish(q2)\n",
    "    if x2:\n",
    "        ret.extend(x2)\n",
    "    return ret\n",
    "#     ret =  [(question.text, x[1]) for x in tg.fill_in_between([wp] + ents) if question.text != x[1]]\n",
    "#     ret = [x for x in ret if len(x[1].split()) < len(x[0].split()) -4][:5]\n",
    "#     return rett"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(parsed_qs, gibberish_both, nsamples=500)\n",
    "expect_false = Expect.eq(0)\n",
    "name = 'Keep entitites, fill in with gibberish'\n",
    "desc = 'Fill in between entitites with BERT, expect result to not be duplicate with original questions'\n",
    "test = DIR(**t, expect=expect_false, name=name, description=desc, capability='NER')\n",
    "# test.run(new_pp)\n",
    "# test.summary(3)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test.summary(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Temporal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Is != used to be"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_nouns = ['player', 'person', 'friend', 'kid', 'candidate']\n",
    "nouns = list(set(professions + other_nouns))\n",
    "t = editor.template(('Is {first_name} {last_name} {a:noun}?', 'Did {first_name} {last_name} use to be {a:noun}?'),\n",
    "                noun=nouns,\n",
    "                adj=adjs,\n",
    "                remove_duplicates=True, \n",
    "                nsamples=1000)\n",
    "name = 'Is person X != Did person use to be X'\n",
    "test = MFT(**t, labels=0, name=name, description='', capability='Temporal')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Is != becoming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(('Is {first_name} {last_name} {a:noun}?', 'Is {first_name} {last_name} becoming {a:noun}?'),\n",
    "                noun=nouns,\n",
    "                adj=adjs,\n",
    "                remove_duplicates=True, \n",
    "                nsamples=1000)\n",
    "name = 'Is person X != Is person becoming X'\n",
    "test = MFT(**t, labels=0, name=name, description='', capability='Temporal')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before != after"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'What was {first_name} {last_name}\\'s life before becoming {a:noun}?',\n",
    "    'What was {first_name} {last_name}\\'s life after becoming {a:noun}?'\n",
    "),\n",
    "                noun=nouns,\n",
    "                adj=adjs,\n",
    "                remove_duplicates=True, \n",
    "                nsamples=1000)\n",
    "name = 'What was person\\'s life before becoming X != What was person\\'s life after becoming X'\n",
    "test = MFT(**t, labels=0, name=name, description='', capability='Temporal')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "action12 = [x for x in editor.suggest('Do you have to {mask} your cat before {mask} it?') if 'kill' not in x[0] and 'kill' not in x[1]][:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Do you have to {a[0]} your {an} before {a[1]} it?',\n",
    "    'Do you have to {a[0]} your {an} after {a[1]} it?'\n",
    "),\n",
    "    an = ['cat', 'dog', 'hamster'],\n",
    "    a=action12,\n",
    "    remove_duplicates=True, \n",
    "   nsamples=1000)\n",
    "name = 'Do you have to X your dog before Y it != Do you have to X your dog after Y it.'\n",
    "test = MFT(**t, labels=0, name=name, description='', capability='Temporal')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "it, normal, OK, there, safe, best, ok, okay, acceptable, reasonable, permissible, legal, proper, wrong, healthy, good, enough, better, important, appropriate, left, allowed, possible, dinner, vegetarian, what, recommended, going, food, illegal, supposed, smart, unhealthy, hard, popular, hot, necessary, vegan, typical, not, free, right, required, fun, time, meant, something, one, kosher, customary, fair, mandatory, available, common, cheaper, alright, easier, suitable, feasible, polite, difficult, cool, certain, sensible, affordable, advisable, fine, fashionable, like, nice, unsafe, public, wise, decent, sufficient, harder, advised, traditional, usual, healthier, anyone, permitted, eating, easy, life, trendy, someone, forbidden, lawful, expensive, safer, new, everyone, compulsory, dangerous, worse, needed, realistic, temptation, essential, adequate, impossible, hungry, rude, risky, considered, prudent, taboo, different, tempting\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Is is {mask} to eat after 10pm?')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "mid = ['normal', 'ok', 'safe', 'dangerous', 'acceptable', 'reasonable', 'proper', 'wrong', 'healthy', 'important']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sleep, drink, eat, work, leave, drive, go, smoke, stay, call, vote, talk, read, retire, visit, write, celebrate, post, do, party, watch, study, exercise, travel, fish, start, stop, walk, play, strike, disappear, be, tweet, live, continue, shop, rise, speak, relax, die, finish, blog, run, move, act, wait, text, cook, return, dance, bed, fly, quit, report, swim, pray, ask, rest, answer, close, gamble, enter, crash, think, marry, come, kill, look, meet, feed, operate, propose, vanish, dress, pee, linger, change, check, remain, dinner, protest, nap, withdraw, wake, fight, begin, publish, arrive, attend, skate, disturb, end, gather, complain, chat, exist, worry, know, cry, shower, cancel, see, open, perform, queue, respond, resign, exit, book, commute, function, avoid, film, buy, awake, binge, snack, clean, vape, park, happen, comment, blackout, occur, cycle, have, paint, vomit, panic, refuse, sit, say, resume, demonstrate, disconnect, kiss, feel, lose, search, hunt, bar, breathe, take, survive, order, vacuum, investigate, emerge, you, workout, me, consume, chill, remember, spend, collapse, date, sex, diet, wear, fart, laugh, waste, join, escape, bake, strip, indulge, practice, phone, cough, barbecue, block, shave, freeze, ski, evacuate, pay, recover, behave, proceed, lie, sunset, fire, conclude, rent, cross, awaken, train, scream, expect, intervene, reopen, rush, pass, ring, debate, stand, argue, listen, us, appear, try, discuss, plan, Google, learn, people, focus, follow, anyone, reconnect, concentrate, notice, hear, tell, consider, research, pause, save, update, them, note\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Is is {mid} to {mask} after 10pm?', mid=mid)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "activity = ['drink', 'sleep', 'drive', 'work', 'eat', 'smoke', 'walk', 'read', 'party', 'talk', 'exercise', 'celebrate', 'text', 'tweet', 'run', 'dance', 'swim', 'cook', 'pray', 'pee', 'rest']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(('Is it {mid} to {activity} before {hour}{ampm}?','Is it {mid} to {activity} after {hour}{ampm}?'),\n",
    "                activity=activity,\n",
    "                mid=mid,\n",
    "                hour=[str(x) for x in range(1, 12)],\n",
    "                ampm=['am', 'pm'],\n",
    "                remove_duplicates=True, \n",
    "                nsamples=1000)\n",
    "name = 'Is it {ok, dangerous, ...} to {smoke, rest, ...} after != before'\n",
    "test = MFT(**t, labels=0, name=name, description='', capability='Temporal')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Negation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "mid2 = mid + ['legal', 'awkward', 'socially acceptable']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "live, work, vote, drive, smoke, travel, be, marry, study, eat, fly, protest, pray, die, fight, speak, stay, shop, exist, campaign, gamble, meet, write, squat, visit, hunt, kill, murder, sleep, demonstrate, perform, drink, surf, serve, fish, gather, move, swim, remain, reside, pee, act, settle, cook, worship, dance, rape, play, strike, party, propose, fire, hide, report, train, march, burn, invest, paint, operate, race, shoot, barbecue, camp, preach, gay, arrive, hate, spy, vacation, migrate, sue, lie, talk, experiment, exercise, tan, search, interview, practice, volunteer, teach, survive, walk, land, buy, date, ski, ask, retire, wait, sit, holiday, communicate, breed, bomb, think, look, rent, diet\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Is it {mid} to {mask} in {country}?', mid=mid2)[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "things = ['work', 'vote', 'travel', 'marry', 'drive', 'study', 'protest', 'campaign', 'fight', 'gamble', 'hunt', 'pray', 'smoke', 'fish', 'murder', 'invest', 'pee', 'march', 'worship', 'volunteer', 'surf', 'shoot', 'dance', 'camp', 'preach', 'spy', 'be gay', 'lie', 'divorce', 'discriminate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = editor.suggest(('How can I become a person who is {mask}', 'How can I become a person who is not {mask}?'))\n",
    "tmp.remove('differently')\n",
    "t = editor.template((\n",
    "    'How can I become {a:x} person?',\n",
    "    'How can I become a person who is not {x}?',\n",
    "    ),\n",
    "    x=tmp,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'How can I become a X person != How can I become a person who is not X' \n",
    "desc = ''\n",
    "test = MFT(**t, labels=0, name=name, capability = 'Negation',\n",
    "          description=desc)\n",
    "# test.run(new_pp, n=500, seed=1)\n",
    "# test.summary(n=5)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(('Is it {mid} to {activity} in {country}?','Is it {mid} not to {activity} in {country}?'),\n",
    "                activity=things,\n",
    "                mid=mid2,\n",
    "                remove_duplicates=True, \n",
    "                nsamples=1000)\n",
    "name = 'Is it {ok, dangerous, ...} to {smoke, rest, ...} in country != Is it {ok, dangerous, ...} not to {smoke, rest, ...} in country'\n",
    "test = MFT(**t, labels=0, name=name, description='', capability='Negation')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'What are things {a:noun} should worry about?',\n",
    "    'What are things {a:noun} should not worry about?',\n",
    "),\n",
    "                noun=nouns,\n",
    "                remove_duplicates=True, \n",
    "                nsamples=1000)\n",
    "name = 'What are things a {noun} should worry about != should not worry about.'\n",
    "test = MFT(**t, labels=0, name=name, description='', capability='Negation')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template([(\n",
    "    'How can I become {a:x[0]} person?',\n",
    "    'How can I become a person who is not {x[1]}?',\n",
    "    ),\n",
    "    (\n",
    "    'How can I become {a:x[1]} person?',\n",
    "    'How can I become a person who is not {x[0]}?',\n",
    "    ),\n",
    "],\n",
    "    unroll=True,\n",
    "    x=antonyms,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'How can I become a X person == How can I become a person who is not antonym(X)' \n",
    "desc = ''\n",
    "test = MFT(**t, labels=1, name=name, capability = 'Negation',\n",
    "          description=desc)\n",
    "# test.run(new_pp, n=500, seed=1)\n",
    "# test.summary(n=3)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Coref"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(\n",
    "    [(\n",
    "        'If {male} and {female} were alone, do you think he would reject her?',\n",
    "        'If {male} and {female} were alone, do you think she would reject him?',\n",
    "    ),\n",
    "        (\n",
    "        'If {female} and {male} were alone, do you think he would reject her?',\n",
    "        'If {female} and {male} were alone, do you think she would reject him?',\n",
    "    )\n",
    "    ],\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000,\n",
    "    unroll=True)\n",
    "name = 'Simple coref: he and she'\n",
    "desc = '' \n",
    "test = MFT(**t, labels=0, name=name, description=desc, capability='Coref')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template(\n",
    "    [(\n",
    "        'If {male} and {female} were married, would his family be happy?',\n",
    "        'If {male} and {female} were married, would {female}\\'s family be happy?',\n",
    "    ),(\n",
    "        'If {male} and {female} were married, would her family be happy?',\n",
    "        'If {male} and {female} were married, would {male}\\'s family be happy?',\n",
    "    ),\n",
    "    ]\n",
    "        ,\n",
    "    unroll=True,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'Simple coref: his and her'\n",
    "desc = '' \n",
    "test = MFT(**t, labels=0, name=name, description=desc, capability='Coref')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test, 'Simple coref: his and her', 'Coref', 'TODO_DESCRIPTION')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SRL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chef, pizza, boxer, player, footballer, athlete, rapper, actor, singer, cook, magician, robot, coach, beer, cyclist, wrestler, drummer, musician, quarterback, hacker, dog, baker, fighter, journalist, restaurant, steak, teacher, doctor, gamer, band, shooter, DJ, football, person, horse, photographer, driver, fisherman, burger, coffee, lawyer, writer, food, dancer, student, wine, artist, man, surgeon, comedian, trainer, VPN, vegetarian, programmer, team, game, vegan, guitarist, goalkeeper, server, guy, cricket, sport, engineer, dentist, waiter, cheese, bartender, fish, chicken, goalie, car, manager, computer, referee, hunter, guitar, basketball, tennis, astronaut, mathematician, sandwich, blogger, accountant, TV, soldier, sniper, job, runner, barbecue, chocolate, scientist, friend, cat, gun, pitcher, judge, AI, philosopher, butcher, painter, golf, camel, translator, piano, worker, farmer, conductor, politician, bike\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Who is the best {mask} in the world?')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "thing = ['chef', 'boxer', 'player', 'footballer', 'athlete', 'rapper', 'actor', 'singer', 'cook', 'magician', 'coach', 'cyclist', 'wrestler', 'drummer', 'musician', 'quarterback', 'hacker', 'baker', 'fighter', 'journalist', 'teacher', 'doctor', 'gamer', 'husband', 'DJ', 'person', 'man', 'woman', 'surgeon', 'comedian', 'trainer', 'programmer', 'guitarist', 'goalkeeper']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "you, YOU, people, we, they, I, readers, You, guys, u, your, fans, ya, others, experts, Americans, scientists, men, voters, some, the, students, analysts, authors, everyone, friends, conservatives, players, critics, Canadians, he, judges, folks, women, all, most, historians, i, coaches, supporters, members, journalists, researchers, celebrities, many, viewers, liberals, participants, audiences, Australians, respondents, editors, Republicans, artists, writers, U, pundits, yo, comedians, gamers, reporters, economists, teachers, parents, veterans, commentators, consumers, anyone, users, independents, kids, pros, millennials, each, individuals, YOUR, those, athletes, doctors, candidates, reviewers, ye, investigators, both, politicians, scholars, not, two, philosophers, atheists, agents, humans, stars, ladies, police, feminists, insiders, yours, any, archaeologists, leaders, Christians, seniors, Democrats, contestants, psychologists, adults, guests, teams, astronomers, competitors, professionals, still, one, academics, scouts, listeners, children, bloggers, Mormons, polls, chefs, wrestlers, fighters, brands, Conservatives, she, legends, strikers, Russians, sports, referees, lawyers, activists, cyclists, captains, Texans, drivers, musicians, rappers, Cardinals, evangelicals, skeptics, schools, DJs, libertarians, girls, riders, Britons, runners, opponents, poets, teenagers, thee, teens, so, professors\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Who do {mask} think is the the best {thing} in the world?', thing=thing)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "subjects = ['you', 'people', 'readers', 'guys', 'fans', 'experts', 'scientists', 'Americans', 'students', 'men', 'voters', 'authors', 'conservatives', 'women', 'Canadians', 'analysts', 'critics', 'judges', 'artists', 'researchers', 'liberals', 'historians', 'Australians', 'journalists', 'Republicans', 'coaches', 'parents', 'kids', 'economists', 'reporters', 'consumers', 'veterans', 'doctors']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best, greatest, worst, top, smartest, finest, strongest, fastest, biggest, toughest, deadliest, coolest, hottest, better, happiest, safest, hardest, great, oldest, elite, richest, leading, premier, brightest, busiest, ultimate, superior, youngest, largest, favorite, Greatest, BEST, highest, professional, easiest, most, foremost, newest, Best, perfect, outstanding, dominant, premiere, first, star, next, only, weakest, wealthiest, quickest\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Who do {subjects} think is the the {mask} {thing} in the world?', thing=thing, subjects=subjects)[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "best = ['best', 'greatest', 'worst', 'top', 'smartest', 'strongest', 'finests', 'happiest', 'coolest', 'richest', 'leading', 'brightest', 'premier', 'ultimate', 'dominant']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Who do {subjects} think is the {best} {thing} in the world?',\n",
    "    'Who is the {best} {thing} in the world according to {subjects}?'\n",
    "),\n",
    "    subjects=subjects,\n",
    "    best=best,\n",
    "    thing=thing,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'Who do X think - Who is the ... according to X'\n",
    "desc = '' \n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='SRL')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "humans, cats, mice, dogs, pigs, people, you, birds, chickens, cows, sheep, rats, fish, bears, rabbits, monkeys, elephants, we, snakes, bees, bats, lions, children, puppies, spiders, babies, ants, insects, they, kittens, dolphins, butterflies, frogs, animals, robots, turtles, ducks, trees, bugs, kids, flies, worms, whales, mosquitoes, crabs, cars, plants, sharks, dinosaurs, beetles, horses, tigers, wolves, primates, cattle, chimpanzees, goats, apes, men, deer, reptiles, rodents, mammals, balls, pets, ponies, carrots, seals, potatoes, apples, mushrooms, dragons, ticks, boys, toys, girls, us, computers, things, stones, women, brains, eggs, bulls, calves, Indians, dwarves, bananas, houses, burgers, flowers, shells, beans, diamonds, coins, trolls, guys, drones, bacteria, Lions\n"
     ]
    }
   ],
   "source": [
    "print(', '.join([str(x) for x in editor.suggest('Are {mask} smaller than {a}?', a=['bananas', 'dogs', 'cars', 'cats', 'elephants'])][:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "things = editor.suggest('Are {mask} smaller than {a}?',a=['bananas', 'dogs', 'cars', 'cats', 'elephants'] )[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "smarter, better, bigger, faster, stronger, cooler, different, smaller, worse, larger, tougher, more, taller, safer, slower, wiser, heavier, weaker, healthier, happier, lighter, nicer, less, older, greater, quicker, cheaper, harder, shorter, cleaner, longer, quieter, brighter, easier, darker, hotter, higher, closer, other, colder, thicker, younger, louder, warmer, lower, sharper, deeper, wider, lesser, softer, broader, simpler, thinner, intelligent, clearer, stranger, important, dangerous, farther, rather, fewer, richer, smoother, finer, differently, smart, tighter, poorer, earlier, sooner, superior, real, related, stupid, wealthier, anymore, inferior, normal, valuable, dumb, interesting, beautiful, weird, narrower, big, further, stricter, strange, human, safe, powerful, rare, MORE, fuller, dead, special, alive, similar, old, newer\n"
     ]
    }
   ],
   "source": [
    "print(', '.join([str(x) for x in editor.suggest('Are {a} {mask} than {a2}?', a=things)][:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "comp = ['better', 'worse', 'cheaper', 'bigger', 'louder', 'longer', 'larger', 'smaller', 'warmer', 'colder', 'thicker', 'lighter', 'heavier']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Order doesn't matter for comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template([\n",
    "    (\n",
    "    'Are {t1} {comp} than {t2}?',\n",
    "    'What is {comp}, {t2} or {t1}?'\n",
    "    ),\n",
    "    (\n",
    "    'Are {t1} {comp} than {t2}?',\n",
    "    'Are {t2} {comp} than {t1}?',\n",
    "    ),\n",
    "    (\n",
    "    'Are {t1} {comp} than {t2}?',\n",
    "    'What is {comp}, {t1} or {t2}?',\n",
    "    )\n",
    "]\n",
    "    ,\n",
    "    t = things,\n",
    "    comp = comp,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'Order does not matter for comparison'\n",
    "desc = '' \n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='SRL')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# editor.template('Is {first_name1} {mask} {first_name2}?', remove_duplicates=True)[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "related, married, close, engaged, talking, lying, connected, speaking, closer, loyal, linked, attracted, going, referring, listening, important, faithful, similar, mean, responding, writing, hostile, true, dead, tied, next, bound, known, indebted, lied, proposing, closest, opposed, truthful, committed, right, up, turning, kind, Married, kin, proposed, attached, abusive, gay, good, real, returning, allergic, straight, available, relevant, talk, attractive, fair, written, nice, dangerous, crazy, new, alive, coming, on, friendly, devoted, cruel, entitled, special, superior, answer, addicted, reacting, out, back, born, supposed, equal, father, drawn, happy, happening, answering, acceptable, nicer, spoken, getting, open, decent, familiar, heir, guilty, talked, mentioned, gone, sympathetic, helpful, insane, down, pregnant, equivalent\n",
      "\n",
      "or, really, and, with, actually, not, an, Really, still, like, /, &, born, a, from, called, real, indeed, now, the, dating, truly, after, even, married, to, marry, named, for, also, marrying, versus, against, his, biologically, without, clone, your, be, meet, remember, Elizabeth, vs, …, father, fucking, King, And, REALLY, is, ever, Prince, behind, NOT, know, leaving, Dating, Young, pregnant, just, killed, another, meeting, AND, nor, Or, love, kill, under, Mary, following, by, John, become, killing, -, so, mean, about, beat, before, Charles, Love, James, find, her, Sir, see, becoming, considered, met, follow, have, Sue, replace, no, Not, into, my, being\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Is {first_name1} {mask} to {first_name2}?', remove_duplicates=True)[:100]))\n",
    "print()\n",
    "print(', '.join(editor.suggest('Is {first_name1} {mask} {first_name2}?', remove_duplicates=True)[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "symmetric = ['dating', 'married to', 'close to', 'engaged to', 'connected to', 'married to', 'friends with', 'related to', 'an acquaintance of']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name1} {s} {first_name2}?',\n",
    "#     'Are {t2} {comp} than {t1}?',\n",
    "#     'What is {comp}, {t1} or {t2}?',\n",
    "    'Is {first_name2} {s} {first_name1}?',\n",
    "),\n",
    "    s = symmetric,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'Order does not matter for symmetric relations'\n",
    "desc = 'e.g. dating, married to, close to, engaged to, etc' \n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='SRL')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Order matters for asymetric relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "asymmetric = ['hurting', 'lying to', 'loyal to', 'faithful to', 'proposing to', 'indebted to', 'abusive to', 'using', 'expecting', 'beating', 'punching', 'raising', 'poisoning', 'protecting', 'kidnapping']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name1} {s} {first_name2}?',\n",
    "#     'Are {t2} {comp} than {t1}?',\n",
    "#     'What is {comp}, {t1} or {t2}?',\n",
    "    'Is {first_name2} {s} {first_name1}?',\n",
    "),\n",
    "    s = asymmetric,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'Order does matter for asymmetric relations'\n",
    "desc = 'e.g. hurting lying to, faithful to, etc'\n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='SRL')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "More traditional SRL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "farm, stake, house, property, company, land, rights, ticket, papers, newspaper, book, island, estate, ranch, boat, horse, paper, shares, tickets, business, gun, books, newspapers, yacht, team, ship, stock, idea, plot, contract, Bible, phone, franchise, shotgun, store, horses, cattle, church, place, game, factory, castle, club, painting, bank, tract, rifle, manuscript, car, campaign, Ark, slaves, plane, school, money, guns, sword, plantation, beer, trust, election, film, ring, building, station, twins, deal, loan, time, tractor, home, letter, IRA, plan, title, watch, story, letters, rest, cows, piece, diary, brewery, charter, diamond, mortgage, hospital, site, post, operation, insurance, railroad, cow, coins, Dell, case, children, wine, telephone, dog\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Did John buy the {mask}?', remove_duplicates=True)[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "obj = ['farm', 'house', 'property', 'company', 'land', 'ticket', 'newspaper', 'book', 'island', 'estate', 'ranch', 'boat', 'horse', 'paper', 'business', 'gun', 'game', 'factory', 'castle', 'painting', 'rifle', 'car', 'school', 'building']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "buy, get, take, sell, have, lose, own, leave, see, keep, want, use, win, need, steal, manage, find, miss, receive, handle, inherit, purchase, abandon, seize, control, run, return, remember, finish, break, enjoy, know, hold, give, drop, claim, recover, move, touch, save, land, pull, on, destroy, share, start, rebuild, crash, call, stop, afford, hit, acquire, join, like, bought, deliver, lead, remove, clear, fix, sign, split, quit, watch, bring, play, survive, left, carry, sold, shoot, maintain, close, grab, settle, reclaim, accept, retain, enter, complete, change, possess, raise, make, wreck, beat, flip, catch, nail, build, forget, ruin, recall, try, retrieve, surrender, notice, mention, got\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Did John {mask} the {obj}?', obj=obj, remove_duplicates=True)[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('buy', 'bought'),\n",
       " ('purchase', 'purchased'),\n",
       " ('sell', 'sold'),\n",
       " ('leave', 'left'),\n",
       " ('own', 'owned'),\n",
       " ('take', 'taken'),\n",
       " ('keep', 'kept'),\n",
       " ('want', 'wanted'),\n",
       " ('lose', 'lost'),\n",
       " ('destroy', 'destroyed'),\n",
       " ('inherit', 'inherited'),\n",
       " ('find', 'found'),\n",
       " ('use', 'used'),\n",
       " ('need', 'needed'),\n",
       " ('receive', 'received'),\n",
       " ('return', 'returned'),\n",
       " ('like', 'liked'),\n",
       " ('enjoy', 'enjoyed'),\n",
       " ('abandon', 'abandoned'),\n",
       " ('manage', 'managed'),\n",
       " ('remember', 'remembered'),\n",
       " ('miss', 'missed'),\n",
       " ('move', 'moved'),\n",
       " ('seize', 'seized'),\n",
       " ('steal', 'stolen')]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pattern\n",
    "import pattern.en\n",
    "verbs = ['buy', 'purchase', 'sell', 'leave', 'own', 'take', 'keep', 'want', 'lose', 'destroy', 'inherit', 'find', 'use', 'need', 'receive', 'return', 'like', 'enjoy', 'abandon', 'manage', 'remember', 'miss', 'move', 'seize', 'steal']\n",
    "a = pattern.en.tenses('stolen')[0]\n",
    "verbs = [(v, pattern.en.conjugate(v, *a)) for v in verbs]\n",
    "verbs[3] = ('leave', 'left')\n",
    "verbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Did {first_name} {verb[0]} the {obj}?',\n",
    "    'Was the {obj} {verb[1]} by {first_name}?'\n",
    "),\n",
    "    verb=verbs,\n",
    "    obj=obj,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'traditional SRL: active / passive swap'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='SRL')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Did {first_name} {verb[0]} the {obj}?',\n",
    "    'Was {first_name} {verb[1]} by the {obj}?'\n",
    "),\n",
    "    verb=verbs,\n",
    "    obj=obj,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'traditional SRL: wrong active / passive swap'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=0, name=name, description=desc, capability='SRL')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "know, marry, kill, love, hate, remember, like, want, and, or, have, date, need, see, meet, tell, Know, find, recognize, choose, understand, forgive, trust, mention, blame, follow, murder, miss, bother, ask, deserve, save, beat, mean, attack, prefer, dislike, resemble, become, survive, support, get, help, visit, kiss, leave, accept, recognise, married, own, notice, be, hurt, call, replace, hit, take, fear, catch, reject, clone, divorce, believe, kidnap, Love, /, Remember, adopt, succeed, question, influence, killed, recall, shoot, contact, raise, respect, rape, use, judge, include, eat, v, owe, confront, join, KILL, despise, Like, sponsor, hire, crush, control, threaten, stalk, destroy, Want, &, defeat, lose\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('Does {first_name} {mask} {first_name2}?', remove_duplicates=True)[:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "pverb = ['love', 'hate', 'like', 'remember', 'recognize', 'trust', 'deserve', 'understand', 'blame', 'dislike', 'prefer', 'follow', 'notice', 'hurt', 'bother', 'support', 'believe', 'accept', 'attack']\n",
    "a = pattern.en.tenses('stolen')[0]\n",
    "pverb = [(v, pattern.en.conjugate(v, *a)) for v in pverb]\n",
    "t = editor.template((\n",
    "    'Does {first_name} {verb[0]} {first_name2}?',\n",
    "    'Is {first_name2} {verb[1]} by {first_name}?',\n",
    "),\n",
    "    verb=pverb,\n",
    "    obj=obj,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'traditional SRL: active / passive swap with people'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='SRL')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "pverb = ['love', 'hate', 'like', 'remember', 'recognize', 'trust', 'deserve', 'understand', 'blame', 'dislike', 'prefer', 'follow', 'notice', 'hurt', 'bother', 'support', 'believe', 'accept', 'attack']\n",
    "a = pattern.en.tenses('stolen')[0]\n",
    "pverb = [(v, pattern.en.conjugate(v, *a)) for v in pverb]\n",
    "t = editor.template((\n",
    "    'Does {first_name} {verb[0]} {first_name2}?',\n",
    "    'Is {first_name} {verb[1]} by {first_name2}?',\n",
    "),\n",
    "    verb=pverb,\n",
    "    obj=obj,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'traditional SRL: wrong active / passive swap with people'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=0, name=name, description=desc, capability='SRL')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name} {last_name} {a:p1} ?',\n",
    "    'Is {first_name} {last_name} {a:p3}?',\n",
    "),\n",
    "    p=professions,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "test = MFT(**t, labels=0)\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name} {last_name} {a:p1} or {a:p2}?',\n",
    "    'Is {first_name} {last_name} simultaneously {a:p3} and {a:p4}?',\n",
    "),\n",
    "    p=professions,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'A or B is not the same as C and D'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=0, name=name, description=desc, capability='Logic')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = editor.template((\n",
    "    'Is {first_name} {last_name} {a:p1} or {a:p2}?',\n",
    "    'Is {first_name} {last_name} simultaneously {a:p1} and {a:p2}?',\n",
    "),\n",
    "    p=professions,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "name = 'A or B is not the same as A and B'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=0, name=name, description=desc, capability='Logic')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "t = editor.template((\n",
    "    'Is {first_name} {last_name} {a:p1} {andor} {a:p2}?',\n",
    "    'Is {first_name} {last_name} {a:p2} {andor} {a:p1}?',\n",
    "),\n",
    "    andor=['and', 'or'],\n",
    "    p=professions,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'A and / or B is the same as B and / or A'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='Logic')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "t = editor.template((\n",
    "    'Is {first_name} {last_name} {a:nat} {p1}?',\n",
    "    'Is {first_name} {last_name} {a:p1} and {nat}?',\n",
    "),\n",
    "    nat=editor.lexicons['nationality'][:20],\n",
    "    p=professions,\n",
    "    remove_duplicates=True, \n",
    "    nsamples=1000)\n",
    "# data = [tuple(np.random.choice(x, 2, replace=False)) for x in data]\n",
    "name = 'a {nationality} {profession} = a {profession} and {nationality}'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='Logic')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reflexivity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(list(all_questions), lambda x:(x, x), nsamples=1000, keep_original=False)\n",
    "name = 'Reflexivity: (q, q) should be duplicate'\n",
    "desc = ''\n",
    "test = MFT(**t, labels=1, name=name, description=desc, capability='Logic')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)\n",
    "# test.summary(n=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Symmetry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Perturb.perturb(qs, lambda x:(x[1], x[0]), nsamples=500, keep_original=True)\n",
    "name = 'Symmetry: f(a, b) = f(b, a)'\n",
    "desc = ''\n",
    "test = INV(t.data, name=name, description=desc, capability='Logic')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# t.data[23]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "def extract_unknown_implications(pairs, labels):\n",
    "    graph = collections.defaultdict(lambda: set())\n",
    "    ls = {}\n",
    "    for x, y in zip(pairs, labels):\n",
    "        graph[x[0]].add(x[1])\n",
    "        graph[x[1]].add(x[0])\n",
    "        t = tuple(sorted(x))\n",
    "        ls[t] = y\n",
    "\n",
    "    d = []\n",
    "    l = []\n",
    "    for x in graph:\n",
    "        if len(graph[x]) == 1:\n",
    "            continue\n",
    "        for y in graph[x]:\n",
    "            t = tuple(sorted((x, y)))\n",
    "    #         print(t, ls[t])\n",
    "        new = list(set([tuple(sorted(a)) for a in itertools.product(list(graph[x]), list(graph[x])) if a[0] != a[1]]))\n",
    "        new = [a for a in new if a not in ls]\n",
    "        for b, c in new:\n",
    "            t1 = tuple(sorted((x, b)))\n",
    "            t2 = tuple(sorted((x, c)))\n",
    "            l1 = ls[t1]\n",
    "            l2 = ls[t2]\n",
    "            if l1 + l2 == 2:\n",
    "                l3 = 1\n",
    "            elif l1 + l2 == 1:\n",
    "                l3 = 0\n",
    "            else:\n",
    "                continue\n",
    "            new_x = [(x, b), (x, c), (b, c)]\n",
    "            new_l = np.array([l1, l2, l3])\n",
    "            d.append(new_x)\n",
    "            l.append(new_l)\n",
    "    return d, l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "data, ls = extract_unknown_implications(qs, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expect_triplet(xs, preds, confs, labels, meta=None):\n",
    "    if (preds[0] + preds[1]) == 2:\n",
    "        if preds[2] != 1:\n",
    "            return np.array([-3, -2, -1])\n",
    "        else:\n",
    "            return np.array([True, True, True])\n",
    "    if (preds[0] + preds[1] == 1) and preds[1] != 0:\n",
    "        if preds[1] != 0:\n",
    "            return np.array([-3, -2, -1])\n",
    "        else:\n",
    "            return np.array([True, True, True])\n",
    "    return None\n",
    "#     if preds[0] != labels[0] or preds[1] != labels[1]:\n",
    "#         return None\n",
    "#     if preds[2] == labels[2]:\n",
    "#         return np.array([True, True, True])\n",
    "#     else:\n",
    "#         return np.array([-3, -2, -1])\n",
    "expect = Expect.testcase(expect_triplet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = 'Testing implications'\n",
    "desc = 'f(x, a) = 1 and f(x, b) = 1 => f(a, b) = 1\\nf(x, a) = 1 and f(x, b) = 0 => f(a, b) = 0\\n Only used (x, a, b) such that (x, a) and (x, b) in val dataset and (a, b) is not.\\n Expectation function filters out examples where f(x, a) or f(x, b) are incorrect'\n",
    "test = DIR(data, expect, labels=ls, name=name, description=desc, capability='Logic')\n",
    "# test.run(new_pp)\n",
    "# test.summary(n=3)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/home/marcotcr/work/checklist/release_data/qqp/qqp_suite.pkl'\n",
    "suite.save(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running the suite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from checklist.pred_wrapper import PredictorWrapper\n",
    "\n",
    "import sys\n",
    "sys.path.append('/home/marcotcr/work/ml-tests/')\n",
    "from mltests import model_wrapper\n",
    "# wraps a model running in a server\n",
    "model = model_wrapper.ModelWrapper()\n",
    "new_pp = PredictorWrapper.wrap_softmax(model.predict_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Modifier: adj\n",
      "Predicting 500 examples\n",
      "Running different adjectives\n",
      "Predicting 500 examples\n",
      "Running Different animals\n",
      "Predicting 500 examples\n",
      "Running Irrelevant modifiers - animals\n",
      "Predicting 500 examples\n",
      "Running Irrelevant modifiers - people\n",
      "Predicting 500 examples\n",
      "Running Irrelevant preamble with different examples.\n",
      "Predicting 500 examples\n",
      "Running Preamble is relevant (different injuries)\n",
      "Predicting 500 examples\n",
      "Running How can I become more {synonym}?\n",
      "Predicting 500 examples\n",
      "Running (question, f(question)) where f(question) replaces synonyms?\n",
      "Predicting 326 examples\n",
      "Running Replace synonyms in real pairs\n",
      "Predicting 696 examples\n",
      "Running How can I become more X != How can I become less X\n",
      "Predicting 500 examples\n",
      "Running How can I become more X = How can I become less antonym(X)\n",
      "Predicting 500 examples\n",
      "Running add one typo\n",
      "Predicting 1500 examples\n",
      "Running contrations\n",
      "Predicting 1468 examples\n",
      "Running (q, paraphrase(q))\n",
      "Predicting 18812 examples\n",
      "Running Product of paraphrases(q1) * paraphrases(q2)\n",
      "Predicting 9640 examples\n",
      "Running same adjectives, different people\n",
      "Predicting 500 examples\n",
      "Running same adjectives, different people v2\n",
      "Predicting 500 examples\n",
      "Running same adjectives, different people v3\n",
      "Predicting 500 examples\n",
      "Running Change same name in both questions\n",
      "Predicting 5423 examples\n",
      "Running Change same location in both questions\n",
      "Predicting 5228 examples\n",
      "Running Change same number in both questions\n",
      "Predicting 4781 examples\n",
      "Running Change first name in one of the questions\n",
      "Predicting 9957 examples\n",
      "Running Change first and last name in one of the questions\n",
      "Predicting 9686 examples\n",
      "Running Change location in one of the questions\n",
      "Predicting 10222 examples\n",
      "Running Change numbers in one of the questions\n",
      "Predicting 9618 examples\n",
      "Running Keep entitites, fill in with gibberish\n",
      "Predicting 4626 examples\n",
      "Running Is person X != Did person use to be X\n",
      "Predicting 500 examples\n",
      "Running Is person X != Is person becoming X\n",
      "Predicting 500 examples\n",
      "Running What was person's life before becoming X != What was person's life after becoming X\n",
      "Predicting 500 examples\n",
      "Running Do you have to X your dog before Y it != Do you have to X your dog after Y it.\n",
      "Predicting 500 examples\n",
      "Running Is it {ok, dangerous, ...} to {smoke, rest, ...} after != before\n",
      "Predicting 500 examples\n",
      "Running How can I become a X person != How can I become a person who is not X\n",
      "Predicting 500 examples\n",
      "Running Is it {ok, dangerous, ...} to {smoke, rest, ...} in country != Is it {ok, dangerous, ...} not to {smoke, rest, ...} in country\n",
      "Predicting 500 examples\n",
      "Running What are things a {noun} should worry about != should not worry about.\n",
      "Predicting 500 examples\n",
      "Running How can I become a X person == How can I become a person who is not antonym(X)\n",
      "Predicting 500 examples\n",
      "Running Simple coref: he and she\n",
      "Predicting 500 examples\n",
      "Running Simple coref: his and her\n",
      "Predicting 500 examples\n",
      "Running Who do X think - Who is the ... according to X\n",
      "Predicting 500 examples\n",
      "Running Order does not matter for comparison\n",
      "Predicting 1500 examples\n",
      "Running Order does not matter for symmetric relations\n",
      "Predicting 500 examples\n",
      "Running Order does matter for asymmetric relations\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: active / passive swap\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: wrong active / passive swap\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: active / passive swap with people\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: wrong active / passive swap with people\n",
      "Predicting 500 examples\n",
      "Running A or B is not the same as C and D\n",
      "Predicting 500 examples\n",
      "Running A or B is not the same as A and B\n",
      "Predicting 500 examples\n",
      "Running A and / or B is the same as B and / or A\n",
      "Predicting 500 examples\n",
      "Running a {nationality} {profession} = a {profession} and {nationality}\n",
      "Predicting 500 examples\n",
      "Running Reflexivity: (q, q) should be duplicate\n",
      "Predicting 500 examples\n",
      "Running Symmetry: f(a, b) = f(b, a)\n",
      "Predicting 1000 examples\n",
      "Running Testing implications\n",
      "Predicting 1500 examples\n"
     ]
    }
   ],
   "source": [
    "suite.run(new_pp, n=500, seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary\n",
      "\n",
      "Modifier: adj\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    374 (74.8%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Is Anthony Peterson an investor?', 'Is Anthony Peterson an active investor?')\n",
      "----\n",
      "0.9 ('Is Jessica Brooks an engineer?', 'Is Jessica Brooks an American engineer?')\n",
      "----\n",
      "1.0 ('Is Andrew Hughes a nurse?', 'Is Andrew Hughes an excellent nurse?')\n",
      "----\n",
      "\n",
      "\n",
      "different adjectives\n",
      "Test cases:      954\n",
      "Test cases run:  500\n",
      "Fails (rate):    1 (0.2%)\n",
      "\n",
      "Example fails:\n",
      "0.7 ('Is Ashley Ramirez Indian?', 'Is Ashley Ramirez American?')\n",
      "----\n",
      "\n",
      "\n",
      "Different animals\n",
      "Test cases:      928\n",
      "Test cases run:  500\n",
      "Fails (rate):    28 (5.6%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Can I feed my rat poop?', 'Can I feed my cat poop?')\n",
      "----\n",
      "0.8 ('Can I feed my lobster formula?', 'Can I feed my spider formula?')\n",
      "----\n",
      "0.7 ('Can I feed my lizard corn?', 'Can I feed my fish corn?')\n",
      "----\n",
      "\n",
      "\n",
      "Irrelevant modifiers - animals\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Irrelevant modifiers - people\n",
      "Test cases:      987\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Irrelevant preamble with different examples.\n",
      "Test cases:      938\n",
      "Test cases run:  500\n",
      "Fails (rate):    494 (98.8%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('My pet dog eats eggs. Is it normal for animals to eat eggs?', 'My pet monkey eats eggs. Is it normal for animals to eat eggs?')\n",
      "----\n",
      "0.0 ('My pet cat eats butter. Is it normal for animals to eat butter?', 'My pet snail eats butter. Is it normal for animals to eat butter?')\n",
      "----\n",
      "0.0 ('My pet squirrel eats sugar. Is it normal for animals to eat sugar?', 'My pet cat eats sugar. Is it normal for animals to eat sugar?')\n",
      "----\n",
      "\n",
      "\n",
      "Preamble is relevant (different injuries)\n",
      "Test cases:      975\n",
      "Test cases run:  500\n",
      "Fails (rate):    129 (25.8%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('I hurt my skin last time I played tennis. Is this a common injury?', 'I hurt my body last time I played tennis. Is this a common injury?')\n",
      "----\n",
      "0.8 ('I hurt my body last time I played soccer. Is it normal to hurt this part of the body?', 'I hurt my knee last time I played soccer. Is it normal to hurt this part of the body?')\n",
      "----\n",
      "0.5 ('I hurt my eye last time I played golf. Should I never play again?', 'I hurt my body last time I played golf. Should I never play again?')\n",
      "----\n",
      "\n",
      "\n",
      "How can I become more X != How can I become less X\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    209 (41.8%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('How can I become less invisible?', 'How can I become more invisible?')\n",
      "----\n",
      "0.7 ('How can I become more stupid?', 'How can I become less stupid?')\n",
      "----\n",
      "0.7 ('How can I become less conservative?', 'How can I become more conservative?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Taxonomy\n",
      "\n",
      "How can I become more {synonym}?\n",
      "Test cases:      6000\n",
      "Test cases run:  500\n",
      "Fails (rate):    109 (21.8%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('How can I become a vocal person?', 'How can I become an outspoken person?')\n",
      "----\n",
      "0.1 ('How can I become more religious?', 'How can I become more spiritual?')\n",
      "----\n",
      "0.3 ('How can I become a modest person?', 'How can I become a humble person?')\n",
      "----\n",
      "\n",
      "\n",
      "(question, f(question)) where f(question) replaces synonyms?\n",
      "Test cases:      326\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Replace synonyms in real pairs\n",
      "Test cases:      251\n",
      "Fails (rate):    42 (16.7%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Does extraterrestrial life exist?', 'Is there any extraterrestrial intelligent life?')\n",
      "0.5 ('Does extraterrestrial life exist?', 'Is there any extraterrestrial smart life?')\n",
      "\n",
      "----\n",
      "1.0 ('Do you know of any psychopaths who have any religious beliefs?', 'Do you know of psychopaths who are religious?')\n",
      "0.0 ('Do you know of any psychopaths who have any religious beliefs?', 'Do you know of psychopaths who are spiritual?')\n",
      "0.4 ('Do you know of any psychopaths who have any spiritual beliefs?', 'Do you know of psychopaths who are religious?')\n",
      "\n",
      "----\n",
      "0.9 ('What is the best way to make a girl happy?', 'How do I make a girl happy?')\n",
      "0.1 ('What is the best way to make a girl happy?', 'How do I make a girl joyful?')\n",
      "0.3 ('What is the best way to make a girl joyful?', 'How do I make a girl happy?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "How can I become more X = How can I become less antonym(X)\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    357 (71.4%)\n",
      "\n",
      "Example fails:\n",
      "0.3 ('How can I become less pessimistic?', 'How can I become more optimistic?')\n",
      "----\n",
      "0.0 ('How can I become more irresponsible?', 'How can I become less responsible?')\n",
      "----\n",
      "0.1 ('How can I become less dependent?', 'How can I become more independent?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Robustness\n",
      "\n",
      "add one typo\n",
      "Test cases:      500\n",
      "Fails (rate):    93 (18.6%)\n",
      "\n",
      "Example fails:\n",
      "0.6 ('Is a 13 cm penis considered small?', 'I am 20 and my penis is 13 cm. is it considered small and will it grow bigger?')\n",
      "0.0 ('Is a 13 cm penis considere dsmall?', 'I am 20 and my penis is 13 cm. is it considered small and will it grow bigger?')\n",
      "0.0 ('Is a 13 cm penis considered small?', 'I am 20 and my penis is 13 cm. is it considereds mall and will it grow bigger?')\n",
      "\n",
      "----\n",
      "0.9 ('Why is my life getting so complicated?', 'Why is my life so complicated?')\n",
      "0.2 ('Wyh is my life getting so complicated?', 'Why is my life so complicated?')\n",
      "\n",
      "----\n",
      "1.0 ('What did Barack Obama do to be awarded with the Peace Nobel Prize?', 'Why did President Obama win the Nobel Peace Prize?')\n",
      "0.3 ('What did Barack Obama do to be awarded with the Peace Nobel Prize?', 'Why did President bOama win the Nobel Peace Prize?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "contrations\n",
      "Test cases:      500\n",
      "Fails (rate):    9 (1.8%)\n",
      "\n",
      "Example fails:\n",
      "0.3 (\"What's the salary after an MBA?\", 'What is the minimum salary that we can expect after MBA?')\n",
      "0.6 ('What is the salary after an MBA?', \"What's the minimum salary that we can expect after MBA?\")\n",
      "\n",
      "----\n",
      "0.7 (\"What's the best time of year to go to Paris?\", 'What is the best month to visit Paris?')\n",
      "0.3 ('What is the best time of year to go to Paris?', \"What's the best month to visit Paris?\")\n",
      "0.4 (\"What's the best time of year to go to Paris?\", \"What's the best month to visit Paris?\")\n",
      "\n",
      "----\n",
      "0.6 ('What are the best books to refer for NEET?', 'Which reference books are good to crack NEET 2017?')\n",
      "0.5 (\"What're the best books to refer for NEET?\", 'Which reference books are good to crack NEET 2017?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "(q, paraphrase(q))\n",
      "Test cases:      200\n",
      "Fails (rate):    141 (70.5%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('How do I join Indian army as a soldier after MBBS?', 'How do I join Indian army as a soldier after MBBS?')\n",
      "0.2 ('In order to join Indian army as a soldier after MBBS, what should you do?', 'In order to join Indian army as a soldier after MBBS, what should I do?')\n",
      "0.3 ('In order to join Indian army as a soldier after MBBS, what should I do?', 'In order to join Indian army as a soldier after MBBS, what should you do?')\n",
      "\n",
      "----\n",
      "1.0 ('How do I make a girl fall in love with you if you are her best friend?', 'How do I make a girl fall in love with you if you are her best friend?')\n",
      "0.2 ('If you want to make a girl fall in love with you if you are her best friend, what should you do?', 'How can you make a girl fall in love with you if you are her best friend?')\n",
      "0.4 ('If I want to make a girl fall in love with you if you are her best friend, what should I do?', 'How can you make a girl fall in love with you if you are her best friend?')\n",
      "\n",
      "----\n",
      "1.0 ('How do I watch Netflix in China?', 'How do I watch Netflix in China?')\n",
      "0.4 ('In order to watch Netflix in China, what should you do?', 'In order to watch Netflix in China, what should I do?')\n",
      "0.5 ('In order to watch Netflix in China, what should I do?', 'In order to watch Netflix in China, what should you do?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Product of paraphrases(q1) * paraphrases(q2)\n",
      "Test cases:      100\n",
      "Fails (rate):    43 (43.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('How do I focus?', 'How can I focus in class?')\n",
      "0.7 ('In order to focus, what should I do?', 'If I want to focus in class, what should I do?')\n",
      "0.6 ('In order to focus, what should you do?', 'If you want to focus in class, what should you do?')\n",
      "\n",
      "----\n",
      "1.0 ('How do I get internship at IITs (UG student)?', 'How do I get a project internships in IITs and IISc?')\n",
      "0.3 ('How can you get internship at IITs (UG student)?', 'If you want to get a project internships in IITs and IISc, what should you do?')\n",
      "0.3 ('How can you get internship at IITs (UG student)?', 'If I want to get a project internships in IITs and IISc, what should I do?')\n",
      "\n",
      "----\n",
      "1.0 ('How can I learn English well in a short time?', 'How can I learn English in a short time?')\n",
      "0.2 ('In order to learn English well in a short time, what should you do?', 'In order to learn English in a short time, what should I do?')\n",
      "0.2 ('How do you learn English well in a short time?', 'If I want to learn English in a short time, what should I do?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "NER\n",
      "\n",
      "same adjectives, different people\n",
      "Test cases:      972\n",
      "Test cases run:  500\n",
      "Fails (rate):    33 (6.6%)\n",
      "\n",
      "Example fails:\n",
      "0.6 ('Is Scott Walker American?', 'Is Brandon Campbell American?')\n",
      "----\n",
      "0.8 ('Is Alexander Perry black?', 'Is Steven Lewis black?')\n",
      "----\n",
      "0.5 ('Is Angela Ramirez black?', 'Is Lauren Edwards black?')\n",
      "----\n",
      "\n",
      "\n",
      "same adjectives, different people v2\n",
      "Test cases:      984\n",
      "Test cases run:  500\n",
      "Fails (rate):    35 (7.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Is Olivia White an atheist?', 'Is Madison White an atheist?')\n",
      "----\n",
      "0.9 ('Is Jose Gray immortal?', 'Is Daniel Gray immortal?')\n",
      "----\n",
      "1.0 ('Is Christina Gomez an atheist?', 'Is Stephanie Gomez an atheist?')\n",
      "----\n",
      "\n",
      "\n",
      "same adjectives, different people v3\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    37 (7.4%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Lisa Bell American?', 'Is Lisa Williams American?')\n",
      "----\n",
      "0.7 ('Is Hannah Scott mad?', 'Is Hannah Jones mad?')\n",
      "----\n",
      "1.0 ('Is Aaron Thomas Christian?', 'Is Aaron Lewis Christian?')\n",
      "----\n",
      "\n",
      "\n",
      "Change same name in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    52 (10.4%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('What stops Bernie Sanders from running as a Republican?', 'Why has Bernie Sanders stopped running?')\n",
      "0.9 ('What stops William Martinez from running as a Republican?', 'Why has William Martinez stopped running?')\n",
      "0.7 ('What stops Christopher Watson from running as a Republican?', 'Why has Christopher Watson stopped running?')\n",
      "\n",
      "----\n",
      "1.0 ('How will Donald Trump benefit India?', 'What Can happen to India if Donald Trump becomes president?')\n",
      "0.0 ('How will Michael Diaz benefit India?', 'What Can happen to India if Michael Diaz becomes president?')\n",
      "0.0 ('How will Joshua Kelly benefit India?', 'What Can happen to India if Joshua Kelly becomes president?')\n",
      "\n",
      "----\n",
      "0.4 (\"What would have happened if John Lennon hadn't been shot?\", 'What if John Lennon were still alive?')\n",
      "1.0 (\"What would have happened if William Martinez hadn't been shot?\", 'What if William Martinez were still alive?')\n",
      "0.9 (\"What would have happened if David Cox hadn't been shot?\", 'What if David Cox were still alive?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change same location in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    43 (8.6%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('How good is Mercedes Benz in India?', 'What is it like to work for Mercedes Benz in India?')\n",
      "0.9 ('How good is Mercedes Benz in Mali?', 'What is it like to work for Mercedes Benz in Mali?')\n",
      "0.7 ('How good is Mercedes Benz in Eritrea?', 'What is it like to work for Mercedes Benz in Eritrea?')\n",
      "\n",
      "----\n",
      "0.0 ('What does Pakistani people think about India?', 'What do pakisthani people think about India?')\n",
      "1.0 ('What does Pakistani people think about France?', 'What do pakisthani people think about France?')\n",
      "1.0 ('What does Pakistani people think about Belgium?', 'What do pakisthani people think about Belgium?')\n",
      "\n",
      "----\n",
      "0.0 ('Is it true that Canada is the best country for immigrants, especially for Indians?', 'With Canada going into a recession, what are the career prospects for immigrants, especially Indians?')\n",
      "0.8 ('Is it true that Botswana is the best country for immigrants, especially for Indians?', 'With Botswana going into a recession, what are the career prospects for immigrants, especially Indians?')\n",
      "0.7 ('Is it true that Cambodia is the best country for immigrants, especially for Indians?', 'With Cambodia going into a recession, what are the career prospects for immigrants, especially Indians?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change same number in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    32 (6.4%)\n",
      "\n",
      "Example fails:\n",
      "0.5 ('What would a sequel to the 2002 movie Blood Work be about?', 'Is there a sequel to the 2002 movie Blood Work?')\n",
      "1.0 ('What would a sequel to the 2337 movie Blood Work be about?', 'Is there a sequel to the 2337 movie Blood Work?')\n",
      "1.0 ('What would a sequel to the 1768 movie Blood Work be about?', 'Is there a sequel to the 1768 movie Blood Work?')\n",
      "\n",
      "----\n",
      "0.3 ('Is it true that humans only use 10% of their brains?', 'Where does the myth that humans only use 10% of their brains come from?')\n",
      "0.6 ('Is it true that humans only use 9% of their brains?', 'Where does the myth that humans only use 9% of their brains come from?')\n",
      "0.6 ('Is it true that humans only use 7% of their brains?', 'Where does the myth that humans only use 7% of their brains come from?')\n",
      "\n",
      "----\n",
      "0.2 ('What business can I start with $5000 capital in cash?', 'What business can I start online to make $5000 a month cashflow?')\n",
      "1.0 ('What business can I start with $5339 capital in cash?', 'What business can I start online to make $5339 a month cashflow?')\n",
      "1.0 ('What business can I start with $5212 capital in cash?', 'What business can I start online to make $5212 a month cashflow?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change first name in one of the questions\n",
      "Test cases:      500\n",
      "After filtering: 275 (55.0%)\n",
      "Fails (rate):    250 (90.9%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Who is winning the presidential election, Trump or Clinton?', 'Who will win the Election? Trump or Clinton?')\n",
      "1.0 ('Who is winning the presidential election, Trump or Clinton?', 'Who will win the Election? Trump or Derek?')\n",
      "1.0 ('Who is winning the presidential election, Trump or Clinton?', 'Who will win the Election? Trump or Sean?')\n",
      "\n",
      "----\n",
      "1.0 ('What are we going to do now that Donald Trump has won the election?', 'Now that Donald Trump is president, what will happen to America?')\n",
      "1.0 ('What are we going to do now that Donald Trump has won the election?', 'Now that Lucas Trump is president, what will happen to America?')\n",
      "1.0 ('What are we going to do now that Donald Trump has won the election?', 'Now that Derek Trump is president, what will happen to America?')\n",
      "\n",
      "----\n",
      "1.0 ('Will people vote for Hillary Clinton because she is a woman?', \"Should people vote for Hillary Clinton just because she's a woman?\")\n",
      "1.0 ('Will people vote for Hillary Clinton because she is a woman?', \"Should people vote for Shannon Clinton just because she's a woman?\")\n",
      "1.0 ('Will people vote for Hillary Clinton because she is a woman?', \"Should people vote for Brooke Clinton just because she's a woman?\")\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change first and last name in one of the questions\n",
      "Test cases:      682\n",
      "Test cases run:  500\n",
      "After filtering: 290 (58.0%)\n",
      "Fails (rate):    92 (31.7%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('As I begin to take in that Donald Trump has been elected President, do you think he may actually be a good president?', 'Could Donald Trump actually be a good president, even better than Hillary Clinton would be?')\n",
      "0.5 ('As I begin to take in that Donald Trump has been elected President, do you think he may actually be a good president?', 'Could Daniel Nelson actually be a good president, even better than Hillary Clinton would be?')\n",
      "\n",
      "----\n",
      "1.0 ('Will Donald Trump or Hillary Clinton win the 2016 US presidential election?', 'Who will be the next president of USA: Hillary Clinton or Donald Trump?')\n",
      "1.0 ('Will Donald Trump or Hillary Clinton win the 2016 US presidential election?', 'Who will be the next president of USA: Melissa Nelson or Donald Trump?')\n",
      "0.9 ('Will Donald Trump or Hillary Clinton win the 2016 US presidential election?', 'Who will be the next president of USA: Hillary Clinton or Michael Martin?')\n",
      "\n",
      "----\n",
      "1.0 ('Are Hillary Clinton supporters unaware that her administration would be 4 more years of Obama?', \"Why do people want Hillary Clinton to be president when it's just going to be four more years of Obama?\")\n",
      "0.6 ('Are Jennifer Butler supporters unaware that her administration would be 4 more years of Obama?', \"Why do people want Hillary Clinton to be president when it's just going to be four more years of Obama?\")\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change location in one of the questions\n",
      "Test cases:      1386\n",
      "Test cases run:  500\n",
      "After filtering: 261 (52.2%)\n",
      "Fails (rate):    43 (16.5%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('What do Indians think about Iran?', 'What do Indians think of Iran and Iranians?')\n",
      "0.7 ('What do Indians think about Iran?', 'What do Indians think of Serbia and Iranians?')\n",
      "0.6 ('What do Indians think about Iran?', 'What do Indians think of Slovak Republic and Iranians?')\n",
      "\n",
      "----\n",
      "1.0 ('Is Chicago turning into another Detroit?', 'Is Chicago in danger of becoming Detroit?')\n",
      "1.0 ('Is Chicago turning into another Detroit?', 'Is Chicago in danger of becoming Crystal Lake?')\n",
      "0.9 ('Is Chicago turning into another Detroit?', 'Is Chicago in danger of becoming St. Cloud?')\n",
      "\n",
      "----\n",
      "1.0 ('Daniel Ek: Why is Spotify not available in India?', \"Why hasn't Daniel Ek brought Spotify to India?\")\n",
      "1.0 ('Daniel Ek: Why is Spotify not available in Tuvalu?', \"Why hasn't Daniel Ek brought Spotify to India?\")\n",
      "1.0 ('Daniel Ek: Why is Spotify not available in India?', \"Why hasn't Daniel Ek brought Spotify to Marshall Islands?\")\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change numbers in one of the questions\n",
      "Test cases:      1500\n",
      "Test cases run:  500\n",
      "After filtering: 271 (54.2%)\n",
      "Fails (rate):    184 (67.9%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Why does 500 and 1000 Rs notes banned by GOI and new notes of 500 and 2000 are issued?', \"Is Modi's decision on demonetization of 500 and 1000 notes welcomed by public?\")\n",
      "1.0 ('Why does 500 and 1199 Rs notes banned by GOI and new notes of 500 and 2000 are issued?', \"Is Modi's decision on demonetization of 500 and 1000 notes welcomed by public?\")\n",
      "1.0 ('Why does 500 and 948 Rs notes banned by GOI and new notes of 500 and 2000 are issued?', \"Is Modi's decision on demonetization of 500 and 1000 notes welcomed by public?\")\n",
      "\n",
      "----\n",
      "1.0 (\"What's your opinion about the decision on removal of 500 and 1000 rupees currency notes?\", \"What is Balaji Viswanathan's take on 500 & 1000 rupees currency notes ban in India?\")\n",
      "1.0 (\"What's your opinion about the decision on removal of 455 and 1000 rupees currency notes?\", \"What is Balaji Viswanathan's take on 500 & 1000 rupees currency notes ban in India?\")\n",
      "1.0 (\"What's your opinion about the decision on removal of 500 and 1000 rupees currency notes?\", \"What is Balaji Viswanathan's take on 500 & 813 rupees currency notes ban in India?\")\n",
      "\n",
      "----\n",
      "1.0 (\"What are loopholes/cons of government's decision of banning 500 and 1000 rupees notes?\", 'What are some loopholes in banning of 500 & 1000rs notes?')\n",
      "1.0 (\"What are loopholes/cons of government's decision of banning 500 and 1000 rupees notes?\", 'What are some loopholes in banning of 473 & 1000rs notes?')\n",
      "1.0 (\"What are loopholes/cons of government's decision of banning 500 and 1000 rupees notes?\", 'What are some loopholes in banning of 445 & 1000rs notes?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Keep entitites, fill in with gibberish\n",
      "Test cases:      500\n",
      "Fails (rate):    165 (33.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('How was Ankara chosen to be the capital of Turkey?', 'What are the difference of air traffic control between the air force and civilian aircraft?')\n",
      "1.0 ('How was Ankara chosen to be the capital of Turkey?', 'How does Ankara see Turkey?')\n",
      "1.0 ('How was Ankara chosen to be the capital of Turkey?', 'How Does Ankara See Turkey?')\n",
      "\n",
      "----\n",
      "0.0 ('Why is Russia leaning towards Pakistan-China by denying long term relationship with India?', 'Is Russia siding with Pakistan?')\n",
      "1.0 ('Why is Russia leaning towards Pakistan-China by denying long term relationship with India?', 'Why are Russia against Pakistan and China against India?')\n",
      "1.0 ('Why is Russia leaning towards Pakistan-China by denying long term relationship with India?', 'Why separates Russia from Pakistan and China from India?')\n",
      "\n",
      "----\n",
      "0.0 ('Why are lawyers being targeted in Balochistan?', 'What are the differences between Shia and Sunni Muslims?')\n",
      "1.0 ('What are the differences between Shia and Sunni Muslims?', 'What distinguishes Shia from Sunni Muslims?')\n",
      "1.0 ('What are the differences between Shia and Sunni Muslims?', 'What distinguishes Shia and Sunni Muslims?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Temporal\n",
      "\n",
      "Is person X != Did person use to be X\n",
      "Test cases:      999\n",
      "Test cases run:  500\n",
      "Fails (rate):    307 (61.4%)\n",
      "\n",
      "Example fails:\n",
      "0.6 ('Is Amanda Flores an advisor?', 'Did Amanda Flores use to be an advisor?')\n",
      "----\n",
      "0.6 ('Is Christina Gonzalez a person?', 'Did Christina Gonzalez use to be a person?')\n",
      "----\n",
      "1.0 ('Is Angela Miller an escort?', 'Did Angela Miller use to be an escort?')\n",
      "----\n",
      "\n",
      "\n",
      "Is person X != Is person becoming X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    74 (14.8%)\n",
      "\n",
      "Example fails:\n",
      "0.6 ('Is Mark King an agent?', 'Is Mark King becoming an agent?')\n",
      "----\n",
      "0.9 ('Is Kyle Long an intern?', 'Is Kyle Long becoming an intern?')\n",
      "----\n",
      "0.8 ('Is Benjamin Nelson an agent?', 'Is Benjamin Nelson becoming an agent?')\n",
      "----\n",
      "\n",
      "\n",
      "What was person's life before becoming X != What was person's life after becoming X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 (\"What was Danielle Bennett's life before becoming an agent?\", \"What was Danielle Bennett's life after becoming an agent?\")\n",
      "----\n",
      "1.0 (\"What was Emily Davis's life before becoming an agent?\", \"What was Emily Davis's life after becoming an agent?\")\n",
      "----\n",
      "0.9 (\"What was Mary Taylor's life before becoming a journalist?\", \"What was Mary Taylor's life after becoming a journalist?\")\n",
      "----\n",
      "\n",
      "\n",
      "Do you have to X your dog before Y it != Do you have to X your dog after Y it.\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Do you have to pet your cat before walking it?', 'Do you have to pet your cat after walking it?')\n",
      "----\n",
      "1.0 ('Do you have to bath your hamster before feeding it?', 'Do you have to bath your hamster after feeding it?')\n",
      "----\n",
      "1.0 ('Do you have to bath your hamster before eating it?', 'Do you have to bath your hamster after eating it?')\n",
      "----\n",
      "\n",
      "\n",
      "Is it {ok, dangerous, ...} to {smoke, rest, ...} after != before\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    486 (97.2%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Is it proper to rest before 3am?', 'Is it proper to rest after 3am?')\n",
      "----\n",
      "0.9 ('Is it healthy to work before 9pm?', 'Is it healthy to work after 9pm?')\n",
      "----\n",
      "0.9 ('Is it wrong to smoke before 4pm?', 'Is it wrong to smoke after 4pm?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Negation\n",
      "\n",
      "How can I become a X person != How can I become a person who is not X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    99 (19.8%)\n",
      "\n",
      "Example fails:\n",
      "0.6 ('How can I become a biased person?', 'How can I become a person who is not biased?')\n",
      "----\n",
      "0.5 ('How can I become a black person?', 'How can I become a person who is not black?')\n",
      "----\n",
      "0.6 ('How can I become a balanced person?', 'How can I become a person who is not balanced?')\n",
      "----\n",
      "\n",
      "\n",
      "Is it {ok, dangerous, ...} to {smoke, rest, ...} in country != Is it {ok, dangerous, ...} not to {smoke, rest, ...} in country\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    125 (25.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is it dangerous to campaign in Lithuania?', 'Is it dangerous not to campaign in Lithuania?')\n",
      "----\n",
      "0.9 ('Is it awkward to camp in Iraq?', 'Is it awkward not to camp in Iraq?')\n",
      "----\n",
      "0.9 ('Is it wrong to campaign in Kyrgyz Republic?', 'Is it wrong not to campaign in Kyrgyz Republic?')\n",
      "----\n",
      "\n",
      "\n",
      "What are things a {noun} should worry about != should not worry about.\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "How can I become a X person == How can I become a person who is not antonym(X)\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    386 (77.2%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('How can I become an impatient person?', 'How can I become a person who is not patient?')\n",
      "----\n",
      "0.1 ('How can I become a defensive person?', 'How can I become a person who is not offensive?')\n",
      "----\n",
      "0.0 ('How can I become a passive person?', 'How can I become a person who is not active?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Coref\n",
      "\n",
      "Simple coref: he and she\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    384 (76.8%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('If Brooke and Dustin were alone, do you think he would reject her?', 'If Brooke and Dustin were alone, do you think she would reject him?')\n",
      "----\n",
      "0.7 ('If Karen and Luis were alone, do you think he would reject her?', 'If Karen and Luis were alone, do you think she would reject him?')\n",
      "----\n",
      "0.7 ('If Jackson and Monica were alone, do you think he would reject her?', 'If Jackson and Monica were alone, do you think she would reject him?')\n",
      "----\n",
      "\n",
      "\n",
      "Simple coref: his and her\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    498 (99.6%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('If Richard and Kathryn were married, would her family be happy?', \"If Richard and Kathryn were married, would Richard's family be happy?\")\n",
      "----\n",
      "1.0 ('If Angel and Emily were married, would her family be happy?', \"If Angel and Emily were married, would Angel's family be happy?\")\n",
      "----\n",
      "0.9 ('If Edward and Katie were married, would his family be happy?', \"If Edward and Katie were married, would Katie's family be happy?\")\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "SRL\n",
      "\n",
      "Who do X think - Who is the ... according to X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    73 (14.6%)\n",
      "\n",
      "Example fails:\n",
      "0.4 ('Who do Americans think is the richest drummer in the world?', 'Who is the richest drummer in the world according to Americans?')\n",
      "----\n",
      "0.4 ('Who do Canadians think is the best person in the world?', 'Who is the best person in the world according to Canadians?')\n",
      "----\n",
      "0.0 ('Who do women think is the leading fighter in the world?', 'Who is the leading fighter in the world according to women?')\n",
      "----\n",
      "\n",
      "\n",
      "Order does not matter for comparison\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    495 (99.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Are drones heavier than boys?', 'What is heavier, boys or drones?')\n",
      "\n",
      "----\n",
      "0.0 ('Are calves bigger than burgers?', 'What is bigger, burgers or calves?')\n",
      "\n",
      "----\n",
      "0.0 ('Are lions colder than deer?', 'What is colder, deer or lions?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Order does not matter for symmetric relations\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    401 (80.2%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Is Joseph close to Taylor?', 'Is Taylor close to Joseph?')\n",
      "----\n",
      "0.1 ('Is Ryan an acquaintance of Nicole?', 'Is Nicole an acquaintance of Ryan?')\n",
      "----\n",
      "0.0 ('Is Anthony married to Daniel?', 'Is Daniel married to Anthony?')\n",
      "----\n",
      "\n",
      "\n",
      "Order does matter for asymmetric relations\n",
      "Test cases:      988\n",
      "Test cases run:  500\n",
      "Fails (rate):    380 (76.0%)\n",
      "\n",
      "Example fails:\n",
      "0.2 ('Is Mary expecting Nicholas?', 'Is Nicholas expecting Mary?')\n",
      "----\n",
      "0.4 ('Is Sara indebted to Isabella?', 'Is Isabella indebted to Sara?')\n",
      "----\n",
      "0.5 ('Is Matthew hurting Nathan?', 'Is Nathan hurting Matthew?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: active / passive swap\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    53 (10.6%)\n",
      "\n",
      "Example fails:\n",
      "0.4 ('Did Christopher lose the book?', 'Was the book lost by Christopher?')\n",
      "----\n",
      "0.2 ('Did Kimberly leave the property?', 'Was the property left by Kimberly?')\n",
      "----\n",
      "0.0 ('Did Joshua miss the book?', 'Was the book missed by Joshua?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: wrong active / passive swap\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    480 (96.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Did Noah like the estate?', 'Was Noah liked by the estate?')\n",
      "----\n",
      "1.0 ('Did Jason move the gun?', 'Was Jason moved by the gun?')\n",
      "----\n",
      "0.8 ('Did David move the school?', 'Was David moved by the school?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: active / passive swap with people\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    340 (68.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Does Timothy attack Brian?', 'Is Brian attacked by Timothy?')\n",
      "----\n",
      "0.0 ('Does Zachary hate Justin?', 'Is Justin hated by Zachary?')\n",
      "----\n",
      "0.3 ('Does Eric remember Victoria?', 'Is Victoria remembered by Eric?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: wrong active / passive swap with people\n",
      "Test cases:      989\n",
      "Test cases run:  500\n",
      "Fails (rate):    489 (97.8%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Does Kelly prefer Adam?', 'Is Kelly preferred by Adam?')\n",
      "----\n",
      "1.0 ('Does David believe Victoria?', 'Is David believed by Victoria?')\n",
      "----\n",
      "0.8 ('Does Aaron follow Hannah?', 'Is Aaron followed by Hannah?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Logic\n",
      "\n",
      "A or B is not the same as C and D\n",
      "Test cases:      828\n",
      "Test cases run:  500\n",
      "Fails (rate):    53 (10.6%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Tyler Gray an economist or an editor?', 'Is Tyler Gray simultaneously an entrepreneur and an accountant?')\n",
      "----\n",
      "0.9 ('Is Danielle Rivera an auditor or an investigator?', 'Is Danielle Rivera simultaneously an investor and an analyst?')\n",
      "----\n",
      "0.8 ('Is Kimberly Howard an accountant or an analyst?', 'Is Kimberly Howard simultaneously an executive and an entrepreneur?')\n",
      "----\n",
      "\n",
      "\n",
      "A or B is not the same as A and B\n",
      "Test cases:      971\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Is Danielle Wood an author or an organizer?', 'Is Danielle Wood simultaneously an author and an organizer?')\n",
      "----\n",
      "0.9 ('Is Eric Brooks a historian or an academic?', 'Is Eric Brooks simultaneously a historian and an academic?')\n",
      "----\n",
      "1.0 ('Is Amanda Evans an artist or an attorney?', 'Is Amanda Evans simultaneously an artist and an attorney?')\n",
      "----\n",
      "\n",
      "\n",
      "A and / or B is the same as B and / or A\n",
      "Test cases:      970\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "a {nationality} {profession} = a {profession} and {nationality}\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Reflexivity: (q, q) should be duplicate\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    3 (0.6%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('What is the breed of this dog? (PIC)', 'What is the breed of this dog? (PIC)')\n",
      "----\n",
      "0.0 (\"What's the English translation of 整序变量\\x87\\x8f?\", \"What's the English translation of 整序变量\\x87\\x8f?\")\n",
      "----\n",
      "0.2 ('How can I download a video from this website?', 'How can I download a video from this website?')\n",
      "----\n",
      "\n",
      "\n",
      "Symmetry: f(a, b) = f(b, a)\n",
      "Test cases:      500\n",
      "Fails (rate):    23 (4.6%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('Would you have sex with Donald Trump?', 'If Donald Trump was gay, would you have sex with him?')\n",
      "0.0 ('If Donald Trump was gay, would you have sex with him?', 'Would you have sex with Donald Trump?')\n",
      "\n",
      "----\n",
      "0.8 ('Would you have sex with Donald Trump?', 'If Donald Trump was gay, would you have sex with him?')\n",
      "0.0 ('If Donald Trump was gay, would you have sex with him?', 'Would you have sex with Donald Trump?')\n",
      "\n",
      "----\n",
      "0.5 ('What do you feel about Donald Trump winning the elections?', 'How do you feel about Donald Trump winning the Republican nomination?')\n",
      "0.9 ('How do you feel about Donald Trump winning the Republican nomination?', 'What do you feel about Donald Trump winning the elections?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Testing implications\n",
      "Test cases:      8328\n",
      "Test cases run:  500\n",
      "After filtering: 455 (91.0%)\n",
      "Fails (rate):    49 (10.8%)\n",
      "\n",
      "Example fails:\n",
      "0.4 ('How can one stop caring about what people say or think about them?', 'How can I stop worrying about what other people think?')\n",
      "1.0 ('How can one stop caring about what people say or think about them?', 'How do I stop caring about what people think about me?')\n",
      "1.0 ('How can I stop worrying about what other people think?', 'How do I stop caring about what people think about me?')\n",
      "\n",
      "----\n",
      "0.0 ('What is the best way to learn networking?', 'How do I learn networking with linux?')\n",
      "1.0 ('What is the best way to learn networking?', 'Which is the best way to learn networking?')\n",
      "0.0 ('How do I learn networking with linux?', 'Which is the best way to learn networking?')\n",
      "\n",
      "----\n",
      "0.0 ('If dark matter strongly interacts with matter then is it what waves in a double slit experiment?', 'Is a sea of massive gravitons what ripples when galaxy clusters collide and is it what waves in a double slit experiment?')\n",
      "1.0 ('If dark matter strongly interacts with matter then is it what waves in a double slit experiment?', \"Is de Broglie's subquantic medium the strongly interacting dark matter which fills 'empty' space? Is it the DM that waves in a double slit experiment?\")\n",
      "0.0 ('Is a sea of massive gravitons what ripples when galaxy clusters collide and is it what waves in a double slit experiment?', \"Is de Broglie's subquantic medium the strongly interacting dark matter which fills 'empty' space? Is it the DM that waves in a double slit experiment?\")\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "suite.summary(n=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Roberta (switched the server)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "suite = TestSuite.from_file('/home/marcotcr/tmp/qqp_suite.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Modifier: adj\n",
      "Predicting 500 examples\n",
      "Running different adjectives\n",
      "Predicting 500 examples\n",
      "Running Different animals\n",
      "Predicting 500 examples\n",
      "Running Irrelevant modifiers - animals\n",
      "Predicting 500 examples\n",
      "Running Irrelevant modifiers - people\n",
      "Predicting 500 examples\n",
      "Running Irrelevant preamble with different examples.\n",
      "Predicting 500 examples\n",
      "Running Preamble is relevant (different injuries)\n",
      "Predicting 500 examples\n",
      "Running How can I become more {synonym}?\n",
      "Predicting 500 examples\n",
      "Running (question, f(question)) where f(question) replaces synonyms?\n",
      "Predicting 326 examples\n",
      "Running Replace synonyms in real pairs\n",
      "Predicting 696 examples\n",
      "Running How can I become more X != How can I become less X\n",
      "Predicting 500 examples\n",
      "Running How can I become more X = How can I become less antonym(X)\n",
      "Predicting 500 examples\n",
      "Running add one typo\n",
      "Predicting 1500 examples\n",
      "Running contrations\n",
      "Predicting 1468 examples\n",
      "Running (q, paraphrase(q))\n",
      "Predicting 18812 examples\n",
      "Running Product of paraphrases(q1) * paraphrases(q2)\n",
      "Predicting 9640 examples\n",
      "Running same adjectives, different people\n",
      "Predicting 500 examples\n",
      "Running same adjectives, different people v2\n",
      "Predicting 500 examples\n",
      "Running same adjectives, different people v3\n",
      "Predicting 500 examples\n",
      "Running Change same name in both questions\n",
      "Predicting 5423 examples\n",
      "Running Change same location in both questions\n",
      "Predicting 5228 examples\n",
      "Running Change same number in both questions\n",
      "Predicting 4781 examples\n",
      "Running Change first name in one of the questions\n",
      "Predicting 9957 examples\n",
      "Running Change first and last name in one of the questions\n",
      "Predicting 9686 examples\n",
      "Running Change location in one of the questions\n",
      "Predicting 10222 examples\n",
      "Running Change numbers in one of the questions\n",
      "Predicting 9618 examples\n",
      "Running Keep entitites, fill in with gibberish\n",
      "Predicting 4626 examples\n",
      "Running Is person X != Did person use to be X\n",
      "Predicting 500 examples\n",
      "Running Is person X != Is person becoming X\n",
      "Predicting 500 examples\n",
      "Running What was person's life before becoming X != What was person's life after becoming X\n",
      "Predicting 500 examples\n",
      "Running Do you have to X your dog before Y it != Do you have to X your dog after Y it.\n",
      "Predicting 500 examples\n",
      "Running Is it {ok, dangerous, ...} to {smoke, rest, ...} after != before\n",
      "Predicting 500 examples\n",
      "Running How can I become a X person != How can I become a person who is not X\n",
      "Predicting 500 examples\n",
      "Running Is it {ok, dangerous, ...} to {smoke, rest, ...} in country != Is it {ok, dangerous, ...} not to {smoke, rest, ...} in country\n",
      "Predicting 500 examples\n",
      "Running What are things a {noun} should worry about != should not worry about.\n",
      "Predicting 500 examples\n",
      "Running How can I become a X person == How can I become a person who is not antonym(X)\n",
      "Predicting 500 examples\n",
      "Running Simple coref: he and she\n",
      "Predicting 500 examples\n",
      "Running Simple coref: his and her\n",
      "Predicting 500 examples\n",
      "Running Who do X think - Who is the ... according to X\n",
      "Predicting 500 examples\n",
      "Running Order does not matter for comparison\n",
      "Predicting 1500 examples\n",
      "Running Order does not matter for symmetric relations\n",
      "Predicting 500 examples\n",
      "Running Order does matter for asymmetric relations\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: active / passive swap\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: wrong active / passive swap\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: active / passive swap with people\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: wrong active / passive swap with people\n",
      "Predicting 500 examples\n",
      "Running A or B is not the same as C and D\n",
      "Predicting 500 examples\n",
      "Running A or B is not the same as A and B\n",
      "Predicting 500 examples\n",
      "Running A and / or B is the same as B and / or A\n",
      "Predicting 500 examples\n",
      "Running a {nationality} {profession} = a {profession} and {nationality}\n",
      "Predicting 500 examples\n",
      "Running Reflexivity: (q, q) should be duplicate\n",
      "Predicting 500 examples\n",
      "Running Symmetry: f(a, b) = f(b, a)\n",
      "Predicting 1000 examples\n",
      "Running Testing implications\n",
      "Predicting 1500 examples\n"
     ]
    }
   ],
   "source": [
    "suite.run(new_pp, n=500, seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary\n",
      "\n",
      "Modifier: adj\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    372 (74.4%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('Is Zachary Brown an intern?', 'Is Zachary Brown a bad intern?')\n",
      "----\n",
      "1.0 ('Is Elizabeth Stewart an engineer?', 'Is Elizabeth Stewart an elite engineer?')\n",
      "----\n",
      "0.8 ('Is Jeffrey Lopez a producer?', 'Is Jeffrey Lopez an experienced producer?')\n",
      "----\n",
      "\n",
      "\n",
      "different adjectives\n",
      "Test cases:      954\n",
      "Test cases run:  500\n",
      "Fails (rate):    1 (0.2%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('Is Kyle Cruz Indian?', 'Is Kyle Cruz an immigrant?')\n",
      "----\n",
      "\n",
      "\n",
      "Different animals\n",
      "Test cases:      928\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Irrelevant modifiers - animals\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Irrelevant modifiers - people\n",
      "Test cases:      987\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Irrelevant preamble with different examples.\n",
      "Test cases:      938\n",
      "Test cases run:  500\n",
      "Fails (rate):    497 (99.4%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('My pet turtle eats soy. Is it normal for animals to eat soy?', 'My pet pig eats soy. Is it normal for animals to eat soy?')\n",
      "----\n",
      "0.0 ('My pet turtle eats chocolate. Is it normal for animals to eat chocolate?', 'My pet rat eats chocolate. Is it normal for animals to eat chocolate?')\n",
      "----\n",
      "0.0 ('My pet squirrel eats meat. Is it normal for animals to eat meat?', 'My pet snake eats meat. Is it normal for animals to eat meat?')\n",
      "----\n",
      "\n",
      "\n",
      "Preamble is relevant (different injuries)\n",
      "Test cases:      975\n",
      "Test cases run:  500\n",
      "Fails (rate):    3 (0.6%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('I hurt my arm last time I played football. Is it normal to hurt this part of the body?', 'I hurt my forearm last time I played football. Is it normal to hurt this part of the body?')\n",
      "----\n",
      "1.0 ('I hurt my skull last time I played tennis. Is this a common injury?', 'I hurt my head last time I played tennis. Is this a common injury?')\n",
      "----\n",
      "0.9 ('I hurt my foot last time I played golf. Is this a common injury?', 'I hurt my feet last time I played golf. Is this a common injury?')\n",
      "----\n",
      "\n",
      "\n",
      "How can I become more X != How can I become less X\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Taxonomy\n",
      "\n",
      "How can I become more {synonym}?\n",
      "Test cases:      6000\n",
      "Test cases run:  500\n",
      "Fails (rate):    190 (38.0%)\n",
      "\n",
      "Example fails:\n",
      "0.3 ('How can I become a brave person?', 'How can I become a courageous person?')\n",
      "----\n",
      "0.0 ('How can I become more joyful?', 'How can I become more happy?')\n",
      "----\n",
      "0.1 ('How can I become more furious?', 'How can I become more angry?')\n",
      "----\n",
      "\n",
      "\n",
      "(question, f(question)) where f(question) replaces synonyms?\n",
      "Test cases:      326\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Replace synonyms in real pairs\n",
      "Test cases:      251\n",
      "Fails (rate):    38 (15.1%)\n",
      "\n",
      "Example fails:\n",
      "0.6 ('What are some little things that make you happy?', \"What's the best thing that make you happy?\")\n",
      "0.1 ('What are some little things that make you joyful?', \"What's the best thing that make you happy?\")\n",
      "0.4 ('What are some little things that make you happy?', \"What's the best thing that make you joyful?\")\n",
      "\n",
      "----\n",
      "0.9 ('How do I make myself more productive/?', 'How do I make myself more productive and happy?')\n",
      "0.2 ('How do I make myself more productive/?', 'How do I make myself more productive and joyful?')\n",
      "\n",
      "----\n",
      "1.0 ('What is being religious?', 'What does being religious mean?')\n",
      "0.0 ('What is being religious?', 'What does being spiritual mean?')\n",
      "0.0 ('What is being spiritual?', 'What does being religious mean?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "How can I become more X = How can I become less antonym(X)\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('How can I become less active?', 'How can I become more passive?')\n",
      "----\n",
      "0.0 ('How can I become less optimistic?', 'How can I become more pessimistic?')\n",
      "----\n",
      "0.1 ('How can I become more bad?', 'How can I become less good?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Robustness\n",
      "\n",
      "add one typo\n",
      "Test cases:      500\n",
      "Fails (rate):    70 (14.0%)\n",
      "\n",
      "Example fails:\n",
      "0.5 ('What is the best Laptop for College in 2016?', 'What would be the best laptop for college?')\n",
      "1.0 ('What is the best Laptop fo rCollege in 2016?', 'What would be the best laptop for college?')\n",
      "0.6 ('What is the best Laptop for College in 2016?', 'What would be the best latpop for college?')\n",
      "\n",
      "----\n",
      "1.0 ('How do I sleep fast?', 'How can I fall asleep fast?')\n",
      "0.0 ('How do I sleep fas?t', 'How can I fall asleep fast?')\n",
      "\n",
      "----\n",
      "1.0 ('Which pollster has been the most accurate for presidential polling in the 2016 election cycle?', 'Which polls were the most consistently accurate during the 2016 campaign?')\n",
      "0.1 ('Wihch pollster has been the most accurate for presidential polling in the 2016 election cycle?', 'Which polls were the most consistently accurate during the 2016 campaign?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "contrations\n",
      "Test cases:      500\n",
      "Fails (rate):    1 (0.2%)\n",
      "\n",
      "Example fails:\n",
      "0.4 ('What is the Police report has been submitted by your respective Thana and is under review at Commissioner Of Police, District Thane City?', 'What does this passport status mean, “Police reports have been submitted by your respective Thana and is under review at the commissioner Office”?')\n",
      "0.6 (\"What's the Police report has been submitted by your respective Thana and is under review at Commissioner Of Police, District Thane City?\", 'What does this passport status mean, “Police reports have been submitted by your respective Thana and is under review at the commissioner Office”?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "(q, paraphrase(q))\n",
      "Test cases:      200\n",
      "Fails (rate):    51 (25.5%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('How do I find the volume equation in chemistry?', 'How do I find the volume equation in chemistry?')\n",
      "0.4 ('If you want to find the volume equation in chemistry, what should you do?', 'How can I find the volume equation in chemistry?')\n",
      "0.4 ('If you want to find the volume equation in chemistry, what should you do?', 'How can you find the volume equation in chemistry?')\n",
      "\n",
      "----\n",
      "1.0 ('How do I know what is my fashion?', 'How do I know what is my fashion?')\n",
      "0.0 ('If I want to know what is my fashion, what should I do?', 'How do you know what is your fashion?')\n",
      "0.0 ('How do you know what is your fashion?', 'If I want to know what is my fashion, what should I do?')\n",
      "\n",
      "----\n",
      "1.0 ('How do I get free silver coins in India?', 'How do I get free silver coins in India?')\n",
      "0.4 ('If you want to get free silver coins in India, what should you do?', 'How can I get free silver coins in India?')\n",
      "0.4 ('In order to get free silver coins in India, what should you do?', 'How can I get free silver coins in India?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Product of paraphrases(q1) * paraphrases(q2)\n",
      "Test cases:      100\n",
      "Fails (rate):    29 (29.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('How can I stop feeling so depressed?', 'How do I stop feeling depressed for no reason?')\n",
      "0.3 ('How do you stop feeling so depressed?', 'If I want to stop feeling depressed for no reason, what should I do?')\n",
      "0.4 ('How can you stop feeling so depressed?', 'If I want to stop feeling depressed for no reason, what should I do?')\n",
      "\n",
      "----\n",
      "0.0 ('How do I focus?', 'How can I focus in class?')\n",
      "0.7 ('If you want to focus, what should you do?', 'In order to focus in class, what should you do?')\n",
      "0.7 ('If I want to focus, what should I do?', 'In order to focus in class, what should I do?')\n",
      "\n",
      "----\n",
      "0.1 ('How do I cheat life?', 'Can you cheat life?')\n",
      "0.6 ('What is a good way to cheat life?', 'Do you think I can cheat life?')\n",
      "0.6 ('What is a good way to cheat life?', 'Do you think I can cheat life?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "NER\n",
      "\n",
      "same adjectives, different people\n",
      "Test cases:      972\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "same adjectives, different people v2\n",
      "Test cases:      984\n",
      "Test cases run:  500\n",
      "Fails (rate):    3 (0.6%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('Is Alyssa Russell Australian?', 'Is Alexis Russell Australian?')\n",
      "----\n",
      "0.7 ('Is Joseph Roberts gay?', 'Is Adam Roberts gay?')\n",
      "----\n",
      "1.0 ('Is Isabella Jackson immortal?', 'Is Elizabeth Jackson immortal?')\n",
      "----\n",
      "\n",
      "\n",
      "same adjectives, different people v3\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    37 (7.4%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Is Maria Gutierrez Christian?', 'Is Maria Gomez Christian?')\n",
      "----\n",
      "0.9 ('Is Christopher Jones mad?', 'Is Christopher Johnson mad?')\n",
      "----\n",
      "0.6 ('Is Sara Nelson gay?', 'Is Sara Jenkins gay?')\n",
      "----\n",
      "\n",
      "\n",
      "Change same name in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    51 (10.2%)\n",
      "\n",
      "Example fails:\n",
      "0.3 ('Is Donald Trump capable of running a country?', 'Is Donald Trump capable of running a country even though he does not have the following?')\n",
      "0.6 ('Is Joshua Garcia capable of running a country?', 'Is Joshua Garcia capable of running a country even though he does not have the following?')\n",
      "\n",
      "----\n",
      "1.0 (\"Why isn't Hillary Clinton in jail?\", 'Could Hillary Clinton actually go to jail?')\n",
      "0.1 (\"Why isn't Amanda Morales in jail?\", 'Could Amanda Morales actually go to jail?')\n",
      "0.1 (\"Why isn't Nicole Martinez in jail?\", 'Could Nicole Martinez actually go to jail?')\n",
      "\n",
      "----\n",
      "0.7 (\"What is the best gift for my boyfriend on Valentine's Day?\", \"What are the Best Gifts for men on Valentine's day?\")\n",
      "0.0 (\"What is the best gift for my boyfriend on Jesus's Day?\", \"What are the Best Gifts for men on Jesus's day?\")\n",
      "0.3 (\"What is the best gift for my boyfriend on Carlos's Day?\", \"What are the Best Gifts for men on Carlos's day?\")\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change same location in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    59 (11.8%)\n",
      "\n",
      "Example fails:\n",
      "0.7 ('Can you buy ballsyic gel/targets in Australia?', 'Can you buy ballistic gel/targets in Australia?')\n",
      "0.2 ('Can you buy ballsyic gel/targets in Netherlands?', 'Can you buy ballistic gel/targets in Netherlands?')\n",
      "0.5 ('Can you buy ballsyic gel/targets in Serbia?', 'Can you buy ballistic gel/targets in Serbia?')\n",
      "\n",
      "----\n",
      "0.1 ('What are some amazing facts about Japan and Japanese culture?', 'What are the biggest misconceptions about Japan and Japanese culture?')\n",
      "0.6 ('What are some amazing facts about Serbia and Japanese culture?', 'What are the biggest misconceptions about Serbia and Japanese culture?')\n",
      "0.6 ('What are some amazing facts about Gibraltar and Japanese culture?', 'What are the biggest misconceptions about Gibraltar and Japanese culture?')\n",
      "\n",
      "----\n",
      "0.0 ('What do you think about China and Chinese people?', 'How do European people think of Chinese people and China?')\n",
      "0.9 ('What do you think about Belgium and Chinese people?', 'How do European people think of Chinese people and Belgium?')\n",
      "0.9 ('What do you think about Gibraltar and Chinese people?', 'How do European people think of Chinese people and Gibraltar?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change same number in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    18 (3.6%)\n",
      "\n",
      "Example fails:\n",
      "0.5 ('What is the smallest four digit number which when divided by 6 leaves reminder 5 and when divided by 5 leaves reminder 3?', 'What is the smallest four-digit number which when divided by 6, leaves a remainder of 5 and when divided by 5 leaves a remainder of 3?')\n",
      "0.8 ('What is the smallest four digit number which when divided by 6 leaves reminder 6 and when divided by 6 leaves reminder 3?', 'What is the smallest four-digit number which when divided by 6, leaves a remainder of 6 and when divided by 6 leaves a remainder of 3?')\n",
      "0.8 ('What is the smallest four digit number which when divided by 6 leaves reminder 6 and when divided by 6 leaves reminder 3?', 'What is the smallest four-digit number which when divided by 6, leaves a remainder of 6 and when divided by 6 leaves a remainder of 3?')\n",
      "\n",
      "----\n",
      "0.2 ('How should I prepare for SSC CGL 2016 without coaching and also want to learn tricks?', 'I want to prepare for ssc cgl 2016. Should I buy study kit from sscportal?')\n",
      "0.7 ('How should I prepare for SSC CGL 1774 without coaching and also want to learn tricks?', 'I want to prepare for ssc cgl 1774. Should I buy study kit from sscportal?')\n",
      "\n",
      "----\n",
      "0.5 ('Who do you think will win the 2016 US presidential elections and whom do you want to win and why?', 'Who will win the 2016 U.S. presidential election and why?')\n",
      "1.0 ('Who do you think will win the 1779 US presidential elections and whom do you want to win and why?', 'Who will win the 1779 U.S. presidential election and why?')\n",
      "1.0 ('Who do you think will win the 2339 US presidential elections and whom do you want to win and why?', 'Who will win the 2339 U.S. presidential election and why?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change first name in one of the questions\n",
      "Test cases:      500\n",
      "After filtering: 268 (53.6%)\n",
      "Fails (rate):    240 (89.6%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Will Donald Trump or Hillary Clinton win the 2016 US presidential election?', 'How likely is it for Donald Trump to win the 2016 US presidential election?')\n",
      "1.0 ('Will Donald Trump or Hillary Clinton win the 2016 US presidential election?', 'How likely is it for William Trump to win the 2016 US presidential election?')\n",
      "1.0 ('Will Donald Trump or Hillary Clinton win the 2016 US presidential election?', 'How likely is it for Logan Trump to win the 2016 US presidential election?')\n",
      "\n",
      "----\n",
      "1.0 ('Will Donald Trump winning the election affect the international student visa?', 'How is the victory of Donald Trump going to affect the international students aspiring to pursue their Masters in US?')\n",
      "1.0 ('Will Donald Trump winning the election affect the international student visa?', 'How is the victory of Aaron Trump going to affect the international students aspiring to pursue their Masters in US?')\n",
      "1.0 ('Will Donald Trump winning the election affect the international student visa?', 'How is the victory of Lucas Trump going to affect the international students aspiring to pursue their Masters in US?')\n",
      "\n",
      "----\n",
      "1.0 ('If Harry Potter were to be made today, without the restriction of only British actors, who should play the major characters?', 'If Harry Potter were to be made today, who should play the major characters ?')\n",
      "1.0 ('If Alex Potter were to be made today, without the restriction of only British actors, who should play the major characters?', 'If Harry Potter were to be made today, who should play the major characters ?')\n",
      "1.0 ('If Jeremiah Potter were to be made today, without the restriction of only British actors, who should play the major characters?', 'If Harry Potter were to be made today, who should play the major characters ?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change first and last name in one of the questions\n",
      "Test cases:      682\n",
      "Test cases run:  500\n",
      "After filtering: 286 (57.2%)\n",
      "Fails (rate):    92 (32.2%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('What is the best Harry Potter movie and why? Is it also your favorite? Why or why not?', 'Which is the best Harry Potter movie?')\n",
      "0.9 ('What is the best Harry Potter movie and why? Is it also your favorite? Why or why not?', 'Which is the best Christopher Watson movie?')\n",
      "\n",
      "----\n",
      "1.0 ('How are the Indians affected if Donald Trump wins the presidential election in the US?', 'What might be the implications and repercussions on India if Donald Trump wins US Presidential elections?')\n",
      "1.0 ('How are the Indians affected if Donald Trump wins the presidential election in the US?', 'What might be the implications and repercussions on India if William Martinez wins US Presidential elections?')\n",
      "1.0 ('How are the Indians affected if Donald Trump wins the presidential election in the US?', 'What might be the implications and repercussions on India if Christopher Watson wins US Presidential elections?')\n",
      "\n",
      "----\n",
      "1.0 ('Is it illegal for Hillary Clinton to become president because her husband served two terms?', 'Is Hillary Clinton ineligible to be POTUS because her spouse has already served 2 terms and Hillary is legally the same person as Bill Clinton?')\n",
      "1.0 ('Is it illegal for Hillary Clinton to become president because her husband served two terms?', 'Is Michelle Clinton ineligible to be POTUS because her spouse has already served 2 terms and Michelle is legally the same person as Bill Clinton?')\n",
      "1.0 ('Is it illegal for Hillary Clinton to become president because her husband served two terms?', 'Is Amy Clinton ineligible to be POTUS because her spouse has already served 2 terms and Amy is legally the same person as Bill Clinton?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change location in one of the questions\n",
      "Test cases:      1386\n",
      "Test cases run:  500\n",
      "After filtering: 259 (51.8%)\n",
      "Fails (rate):    46 (17.8%)\n",
      "\n",
      "Example fails:\n",
      "0.9 (\"Why didn't NATO attack India when India liberated Goa from Portuguese Colonialism?\", \"Why didn't NATO come to the assistance of Portugal to prevent Indian annexation of Portugese Goa in 1961?\")\n",
      "0.8 (\"Why didn't NATO attack Northern Mariana Islands when Northern Mariana Islands liberated Goa from Portuguese Colonialism?\", \"Why didn't NATO come to the assistance of Portugal to prevent Indian annexation of Portugese Goa in 1961?\")\n",
      "0.7 (\"Why didn't NATO attack Faroe Islands when Faroe Islands liberated Goa from Portuguese Colonialism?\", \"Why didn't NATO come to the assistance of Portugal to prevent Indian annexation of Portugese Goa in 1961?\")\n",
      "\n",
      "----\n",
      "1.0 ('How is the New Bhim APP going to change India?', 'What is your review of BHIM App? How is it going to change India?')\n",
      "1.0 ('How is the New Bhim APP going to change Samoa?', 'What is your review of BHIM App? How is it going to change India?')\n",
      "0.9 ('How is the New Bhim APP going to change India?', 'What is your review of BHIM App? How is it going to change Samoa?')\n",
      "\n",
      "----\n",
      "0.8 ('When will India take back PoK from Pakistan?', 'Why India not demand POK (pakistan occupied Kashmir) from Pakistan?')\n",
      "0.9 ('When will India take back PoK from Pakistan?', 'Why Samoa not demand POK (pakistan occupied Kashmir) from Pakistan?')\n",
      "0.8 ('When will India take back PoK from Pakistan?', 'Why Greenland not demand POK (pakistan occupied Kashmir) from Pakistan?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change numbers in one of the questions\n",
      "Test cases:      1500\n",
      "Test cases run:  500\n",
      "After filtering: 268 (53.6%)\n",
      "Fails (rate):    177 (66.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Who will win the next 2019 general elections in India?', 'Who will win the next Lok Sabha elections in India in 2019?')\n",
      "0.7 ('Who will win the next 2048 general elections in India?', 'Who will win the next Lok Sabha elections in India in 2019?')\n",
      "\n",
      "----\n",
      "1.0 ('Taxes in India: Can donations to trust as a part of CSR contribution, be claimed u/s 80G of income tax act, 1961?', 'Taxes in India: Can donations to trust as a part of CSR contribution, be claimed u/s 80G of income tax act 1961?')\n",
      "1.0 ('Taxes in India: Can donations to trust as a part of CSR contribution, be claimed u/s 80G of income tax act, 1961?', 'Taxes in India: Can donations to trust as a part of CSR contribution, be claimed u/s 80G of income tax act 1961?')\n",
      "1.0 ('Taxes in India: Can donations to trust as a part of CSR contribution, be claimed u/s 80G of income tax act, 1961?', 'Taxes in India: Can donations to trust as a part of CSR contribution, be claimed u/s 80G of income tax act 1961?')\n",
      "\n",
      "----\n",
      "1.0 ('How does banning 500 and 1000 rupee notes help to control black money?', 'How will demonetization of 500 and 1000 rupee notes stop corruption?')\n",
      "1.0 ('How does banning 508 and 1000 rupee notes help to control black money?', 'How will demonetization of 500 and 1000 rupee notes stop corruption?')\n",
      "1.0 ('How does banning 469 and 1000 rupee notes help to control black money?', 'How will demonetization of 500 and 1000 rupee notes stop corruption?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Keep entitites, fill in with gibberish\n",
      "Test cases:      500\n",
      "Fails (rate):    181 (36.2%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Why does the Amazon Instant Video app for iPhone only work on Wi-Fi?', 'How can you use Amazon Instant Video on a Galaxy S4?')\n",
      "0.7 ('Why does the Amazon Instant Video app for iPhone only work on Wi-Fi?', 'Why is the Amazon Instant Video on iPhone using Wi-Fi?')\n",
      "0.7 ('Why does the Amazon Instant Video app for iPhone only work on Wi-Fi?', 'Why does the Amazon Instant Video on iPhone require Wi-Fi?')\n",
      "\n",
      "----\n",
      "1.0 ('I want to get into Google summer code 2017.How do I prepare?', 'What should I do to get an entry in the Google Summer of Code contest?')\n",
      "0.6 ('I want to get into Google summer code 2017.How do I prepare?', 'I the Google summer?')\n",
      "\n",
      "----\n",
      "1.0 ('How can I earn money through YouTube?', 'How do I make money with YouTube?')\n",
      "0.7 ('How do I make money with YouTube?', 'How About YouTube?')\n",
      "0.7 ('How can I earn money through YouTube?', 'How About YouTube?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Temporal\n",
      "\n",
      "Is person X != Did person use to be X\n",
      "Test cases:      999\n",
      "Test cases run:  500\n",
      "Fails (rate):    486 (97.2%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Sophia Lopez a candidate?', 'Did Sophia Lopez use to be a candidate?')\n",
      "----\n",
      "0.9 ('Is Noah Mitchell an adviser?', 'Did Noah Mitchell use to be an adviser?')\n",
      "----\n",
      "0.9 ('Is Anthony Bell a photographer?', 'Did Anthony Bell use to be a photographer?')\n",
      "----\n",
      "\n",
      "\n",
      "Is person X != Is person becoming X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    97 (19.4%)\n",
      "\n",
      "Example fails:\n",
      "0.7 ('Is Joshua Morales an assistant?', 'Is Joshua Morales becoming an assistant?')\n",
      "----\n",
      "0.8 ('Is Amber Allen an intern?', 'Is Amber Allen becoming an intern?')\n",
      "----\n",
      "0.8 ('Is Amber Cook an intern?', 'Is Amber Cook becoming an intern?')\n",
      "----\n",
      "\n",
      "\n",
      "What was person's life before becoming X != What was person's life after becoming X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Do you have to X your dog before Y it != Do you have to X your dog after Y it.\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    307 (61.4%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Do you have to bath your hamster before feeding it?', 'Do you have to bath your hamster after feeding it?')\n",
      "----\n",
      "0.7 ('Do you have to scratch your dog before feeding it?', 'Do you have to scratch your dog after feeding it?')\n",
      "----\n",
      "1.0 ('Do you have to name your dog before naming it?', 'Do you have to name your dog after naming it?')\n",
      "----\n",
      "\n",
      "\n",
      "Is it {ok, dangerous, ...} to {smoke, rest, ...} after != before\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    160 (32.0%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('Is it proper to pray before 8pm?', 'Is it proper to pray after 8pm?')\n",
      "----\n",
      "1.0 ('Is it wrong to text before 11am?', 'Is it wrong to text after 11am?')\n",
      "----\n",
      "0.8 ('Is it healthy to pee before 7am?', 'Is it healthy to pee after 7am?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Negation\n",
      "\n",
      "How can I become a X person != How can I become a person who is not X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Is it {ok, dangerous, ...} to {smoke, rest, ...} in country != Is it {ok, dangerous, ...} not to {smoke, rest, ...} in country\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    2 (0.4%)\n",
      "\n",
      "Example fails:\n",
      "0.7 ('Is it healthy to discriminate in Seychelles?', 'Is it healthy not to discriminate in Seychelles?')\n",
      "----\n",
      "0.7 ('Is it healthy to discriminate in Seychelles?', 'Is it healthy not to discriminate in Seychelles?')\n",
      "----\n",
      "\n",
      "\n",
      "What are things a {noun} should worry about != should not worry about.\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "How can I become a X person == How can I become a person who is not antonym(X)\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    422 (84.4%)\n",
      "\n",
      "Example fails:\n",
      "0.5 ('How can I become a negative person?', 'How can I become a person who is not positive?')\n",
      "----\n",
      "0.4 ('How can I become an insecure person?', 'How can I become a person who is not secure?')\n",
      "----\n",
      "0.0 ('How can I become a progressive person?', 'How can I become a person who is not conservative?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Coref\n",
      "\n",
      "Simple coref: he and she\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    484 (96.8%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('If Nicholas and Katie were alone, do you think he would reject her?', 'If Nicholas and Katie were alone, do you think she would reject him?')\n",
      "----\n",
      "0.9 ('If Megan and Aaron were alone, do you think he would reject her?', 'If Megan and Aaron were alone, do you think she would reject him?')\n",
      "----\n",
      "0.7 ('If Timothy and Melissa were alone, do you think he would reject her?', 'If Timothy and Melissa were alone, do you think she would reject him?')\n",
      "----\n",
      "\n",
      "\n",
      "Simple coref: his and her\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('If Jesus and Tracy were married, would his family be happy?', \"If Jesus and Tracy were married, would Tracy's family be happy?\")\n",
      "----\n",
      "1.0 ('If Matthew and Stephanie were married, would his family be happy?', \"If Matthew and Stephanie were married, would Stephanie's family be happy?\")\n",
      "----\n",
      "1.0 ('If Luke and Rebecca were married, would his family be happy?', \"If Luke and Rebecca were married, would Rebecca's family be happy?\")\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "SRL\n",
      "\n",
      "Who do X think - Who is the ... according to X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Order does not matter for comparison\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Are deer longer than Lions?', 'What is longer, Lions or deer?')\n",
      "0.0 ('Are deer longer than Lions?', 'Are Lions longer than deer?')\n",
      "\n",
      "----\n",
      "0.0 ('Are balls better than spiders?', 'What is better, spiders or balls?')\n",
      "0.0 ('Are balls better than spiders?', 'Are spiders better than balls?')\n",
      "\n",
      "----\n",
      "0.0 ('Are brains bigger than butterflies?', 'What is bigger, butterflies or brains?')\n",
      "0.0 ('Are brains bigger than butterflies?', 'Are butterflies bigger than brains?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Order does not matter for symmetric relations\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Is Tyler married to Erin?', 'Is Erin married to Tyler?')\n",
      "----\n",
      "0.0 ('Is Richard married to Amanda?', 'Is Amanda married to Richard?')\n",
      "----\n",
      "0.0 ('Is Melissa an acquaintance of Andrea?', 'Is Andrea an acquaintance of Melissa?')\n",
      "----\n",
      "\n",
      "\n",
      "Order does matter for asymmetric relations\n",
      "Test cases:      988\n",
      "Test cases run:  500\n",
      "Fails (rate):    499 (99.8%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Is Ryan loyal to Zachary?', 'Is Zachary loyal to Ryan?')\n",
      "----\n",
      "0.0 ('Is Christopher expecting Jessica?', 'Is Jessica expecting Christopher?')\n",
      "----\n",
      "0.0 ('Is Abigail abusive to Laura?', 'Is Laura abusive to Abigail?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: active / passive swap\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    116 (23.2%)\n",
      "\n",
      "Example fails:\n",
      "0.3 ('Did Amber want the ticket?', 'Was the ticket wanted by Amber?')\n",
      "----\n",
      "0.3 ('Did Amy miss the factory?', 'Was the factory missed by Amy?')\n",
      "----\n",
      "0.2 ('Did Ryan lose the paper?', 'Was the paper lost by Ryan?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: wrong active / passive swap\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Did Benjamin own the book?', 'Was Benjamin owned by the book?')\n",
      "----\n",
      "1.0 ('Did Michelle move the property?', 'Was Michelle moved by the property?')\n",
      "----\n",
      "1.0 ('Did Adam leave the painting?', 'Was Adam left by the painting?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: active / passive swap with people\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    498 (99.6%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Does Ethan love Natalie?', 'Is Natalie loved by Ethan?')\n",
      "----\n",
      "0.0 ('Does Taylor notice Laura?', 'Is Laura noticed by Taylor?')\n",
      "----\n",
      "0.0 ('Does Richard love Emma?', 'Is Emma loved by Richard?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: wrong active / passive swap with people\n",
      "Test cases:      989\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 ('Does Alexander attack Jason?', 'Is Alexander attacked by Jason?')\n",
      "----\n",
      "1.0 ('Does Christopher deserve Dylan?', 'Is Christopher deserved by Dylan?')\n",
      "----\n",
      "1.0 ('Does Laura accept Jennifer?', 'Is Laura accepted by Jennifer?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Logic\n",
      "\n",
      "A or B is not the same as C and D\n",
      "Test cases:      828\n",
      "Test cases run:  500\n",
      "Fails (rate):    40 (8.0%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('Is Anthony Long an artist or an agent?', 'Is Anthony Long simultaneously an adviser and an escort?')\n",
      "----\n",
      "0.7 ('Is Hannah Murphy an actor or an entrepreneur?', 'Is Hannah Murphy simultaneously an intern and an agent?')\n",
      "----\n",
      "0.8 ('Is Jacob Long an advisor or an educator?', 'Is Jacob Long simultaneously an adviser and an investigator?')\n",
      "----\n",
      "\n",
      "\n",
      "A or B is not the same as A and B\n",
      "Test cases:      971\n",
      "Test cases run:  500\n",
      "Fails (rate):    498 (99.6%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Amber Brooks an educator or a historian?', 'Is Amber Brooks simultaneously an educator and a historian?')\n",
      "----\n",
      "0.8 ('Is Emma Peterson an artist or an editor?', 'Is Emma Peterson simultaneously an artist and an editor?')\n",
      "----\n",
      "0.9 ('Is Andrea Phillips an actress or an architect?', 'Is Andrea Phillips simultaneously an actress and an architect?')\n",
      "----\n",
      "\n",
      "\n",
      "A and / or B is the same as B and / or A\n",
      "Test cases:      970\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "a {nationality} {profession} = a {profession} and {nationality}\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Reflexivity: (q, q) should be duplicate\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    5 (1.0%)\n",
      "\n",
      "Example fails:\n",
      "0.2 (\"If my mother's mother get pregnant for my father what is the child to me?\", \"If my mother's mother get pregnant for my father what is the child to me?\")\n",
      "----\n",
      "0.0 ('What universities does W&T offshore recruit new grads from? What majors are they looking for?', 'What universities does W&T offshore recruit new grads from? What majors are they looking for?')\n",
      "\n",
      "----\n",
      "0.0 ('What universities does W&T offshore recruit new grads from? What majors are they looking for?', 'What universities does W&T offshore recruit new grads from? What majors are they looking for?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Symmetry: f(a, b) = f(b, a)\n",
      "Test cases:      500\n",
      "Fails (rate):    10 (2.0%)\n",
      "\n",
      "Example fails:\n",
      "0.3 ('How about sociology as an optional subject?', 'Is sociology an optional subject?')\n",
      "0.5 ('Is sociology an optional subject?', 'How about sociology as an optional subject?')\n",
      "\n",
      "----\n",
      "0.7 ('What are things that make Indians happy?', \"Why shouldn't we as what are the things that make Indians happy?\")\n",
      "0.4 (\"Why shouldn't we as what are the things that make Indians happy?\", 'What are things that make Indians happy?')\n",
      "\n",
      "----\n",
      "0.4 ('What is the requirement to study medicine in germany?', 'How do I study medicine in germany?')\n",
      "0.6 ('How do I study medicine in germany?', 'What is the requirement to study medicine in germany?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Testing implications\n",
      "Test cases:      8328\n",
      "Test cases run:  500\n",
      "After filtering: 456 (91.2%)\n",
      "Fails (rate):    44 (9.6%)\n",
      "\n",
      "Example fails:\n",
      "0.0 ('Which is better Canon 700D or Nikon D5200?', 'Canon 1300D and Nikon D5200, which one is better?')\n",
      "0.9 ('Which is better Canon 700D or Nikon D5200?', 'Which is better, Canon 700D or Nikon D5200, for photography and short documentaries?')\n",
      "0.0 ('Canon 1300D and Nikon D5200, which one is better?', 'Which is better, Canon 700D or Nikon D5200, for photography and short documentaries?')\n",
      "\n",
      "----\n",
      "1.0 ('How do I hack WiFi passwords?', 'How do I bypass a wifi password?')\n",
      "0.8 ('How do I hack WiFi passwords?', 'How do I hack a wifi?')\n",
      "0.2 ('How do I bypass a wifi password?', 'How do I hack a wifi?')\n",
      "\n",
      "----\n",
      "0.0 ('What is it like to work at Google?', '?')\n",
      "1.0 ('What is it like to work at Google?', 'What’s it like working at Google?')\n",
      "0.0 ('?', 'What’s it like working at Google?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "suite.summary(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "checklist",
   "language": "python",
   "name": "checklist"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}