cross_validation_gis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.corpus.reader import ConllCorpusReader\n",
    "import nltk.chunk.named_entity\n",
    "from nltk.classify import MaxentClassifier\n",
    "import os\n",
    "import fnmatch\n",
    "from random import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_classifier_builder(self, train):\n",
    "        print(\"The algorithm GIS is being used!\")\n",
    "        return MaxentClassifier.train(\n",
    "            train, algorithm=\"GIS\", gaussian_prior_sigma=1, trace=3, max_iter=10\n",
    "        )\n",
    "\n",
    "nltk.chunk.named_entity.NEChunkParserTagger._classifier_builder = new_classifier_builder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement cv myself\n",
    "# inside the loop: training, predict\n",
    "# define scoring function with the previous function which says a label is right or not\n",
    "# k fold -> list of data splits -> chunked_sents \n",
    "# not round number of samples? divide them. three folds with 9 samples instead of one fold with 7\n",
    "\n",
    "# def filecount(dir):\n",
    "#     path, dirs, files = next(os.walk(dir))\n",
    "#     file_count = len(files)\n",
    "#     return file_count\n",
    "\n",
    "def filecount(dir):\n",
    "    num_files = len(fnmatch.filter(os.listdir(dir),'eng.*')) + len(fnmatch.filter(os.listdir(dir),'deu.*'))\n",
    "    return num_files\n",
    "\n",
    "def cross_validation(k, language, dataform, datadir, resultsdir):\n",
    "    if language != 'eng' and language != 'deu': raise ValueError('Language parameter incorrect.')\n",
    "    columnlist = ['IOB Accuracy','Precision:','Recall','F-Measure', 'True Positives', 'False Positives', 'False Negatives']\n",
    "    df = pd.DataFrame(columns= columnlist)\n",
    "    filesum = filecount(datadir)\n",
    "    samplels = []\n",
    "    for i in range(filesum):\n",
    "        with open(datadir + '/' + language + '.' + str(i)) as ele:\n",
    "            samplels.append(ele.read())\n",
    "    samplels = list(filter(None, samplels))\n",
    "    shuffle(samplels)\n",
    "    print('Cross validation for ' + str(len(samplels)) + ' samples has begun')\n",
    "    foldsize = len(samplels) // k\n",
    "    num_extra_bins = len(samplels) % k # the number of extra sized bins\n",
    "    foldsizes = [foldsize + (i < num_extra_bins) for i in range(k)]\n",
    "    trainls = ['' for i in range(k)]\n",
    "    for i in range(k):\n",
    "        for j in range(foldsizes[i]):\n",
    "            trainls[i] += samplels[j+ sum(foldsizes[0:i])] + '\\n'\n",
    "    for i in range(k):\n",
    "        print(\"Training for iteration \" + str(i+1) + ' started')\n",
    "        trainingls = trainls[:i] + trainls[i+1:]\n",
    "        testls = trainls[i]\n",
    "        with open(datadir + '/CV' + '/iteration' + str(i) + '.train', 'w') as tr:\n",
    "            for j in range(len(trainingls)):\n",
    "                tr.write(trainingls[j])\n",
    "        if dataform == 'conllpos':\n",
    "            training = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.train', ['words', 'chunk', 'pos'])\n",
    "        elif language == 'deu' and dataform == 'conll':\n",
    "            training = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.train', ['words', 'ignore', 'pos', 'ignore', 'chunk'])\n",
    "        elif language == 'eng' and dataform == 'conll':\n",
    "            training = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.train', ['words', 'pos', 'ignore', 'chunk'])\n",
    "        elif language == 'deu' and dataform == 'germeval':\n",
    "            training = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.train', ['ignore', 'words', 'chunk', 'ignore', 'pos'])\n",
    "        else: raise ValueError(\"This combination of data language and data form does not exist.\")\n",
    "        ne_chunker = nltk.chunk.named_entity.NEChunkParser(training.chunked_sents())\n",
    "        print(\"Training for iteration \" + str(i+1) + ' completed')\n",
    "\n",
    "        with open(datadir + '/CV' + '/iteration' + str(i) + '.test', 'w') as te:\n",
    "            te.write(testls)\n",
    "        if dataform == 'conllpos':\n",
    "            test = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.test', ['words', 'chunk', 'pos'])\n",
    "        elif language == 'deu' and dataform == 'conll':\n",
    "            test = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.test', ['words', 'ignore', 'pos', 'ignore', 'chunk'])\n",
    "        elif language == 'eng' and dataform == 'conll':\n",
    "            test = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.test', ['words', 'pos', 'ignore', 'chunk'])\n",
    "        elif language == 'deu' and dataform == 'germeval':\n",
    "            test = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.test', ['ignore', 'words', 'chunk', 'ignore', 'pos'])\n",
    "        else: raise ValueError(\"This combination of data language and data form does not exist.\")\n",
    "        print(\"Evaluation for iteration \" + str(i+1) + ' started')\n",
    "        results = ne_chunker.evaluate(test.chunked_sents())\n",
    "        bufferdf = pd.DataFrame([[results.accuracy(),results.precision(),results.recall(),results.f_measure(),results._tp_num, results._fp_num, results._fn_num]], columns = columnlist, index = ['Iteration' + str(i + 1)])\n",
    "        df = df.append(bufferdf)\n",
    "        print(\"Evaluation for iteration \" + str(i+1) + ' completed')\n",
    "    df.to_csv(resultsdir)\n",
    "    print('results exported to: ' + resultsdir)\n",
    "    print('Cross validation complete.')\n",
    "\n",
    "        #comments:\n",
    "        #read the files in a list \n",
    "        #put the texts to each fold together\n",
    "        #train\n",
    "        #predict, use the previous scoring function\n",
    "        #export\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for iteration 1\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-1pdvrsdc']\n",
      "Training for iteration 2\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-exri4mkp']\n",
      "Training for iteration 3\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-9exchqnj']\n",
      "Training for iteration 4\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-9ybh6fka']\n",
      "Training for iteration 5\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-atskr0eb']\n",
      "Training for iteration 6\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-ayf391a2']\n",
      "Training for iteration 7\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-6_volivx']\n",
      "Training for iteration 8\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-yghwg0g0']\n",
      "Training for iteration 9\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-1ou2ft7b']\n",
      "Training for iteration 10\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-cjrtwgtv']\n",
      "results exported to: ../../results/2nd.csv\n"
     ]
    }
   ],
   "source": [
    "# print('10-fold, document-level, Conll2003, German, nltk NEChunkParser, GIS')\n",
    "# %time cross_validation(10, 'deu', 'conll', '../../data/conll2003/splitdeu', '../../results/13th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10-fold, sentence-level, LegalER coarse-grained, German, nltk NEChunkParser, megam\n",
      "Cross validation for 66723 samples has begun\n",
      "Training for iteration 1 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-in_z6_78']\n",
      "\n",
      "None\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "megam command failed!",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-209b91047a61>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'10-fold, sentence-level, LegalER coarse-grained, German, nltk NEChunkParser, megam'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mcross_validation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'deu'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'conllpos'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'../../data/LegalER/dataset_courts/coarse-grained/split'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'../../results/2nd.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-2-e3468eea4615>\u001b[0m in \u001b[0;36mcross_validation\u001b[0;34m(k, language, dataform, datadir, resultsdir)\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0mtraining\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mConllCorpusReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatadir\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/CV'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'iteration'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'.train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'words'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'chunk'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'pos'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"This combination of data language and data form does not exist.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mne_chunker\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_entity\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNEChunkParser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunked_sents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training for iteration \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' completed'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, train)\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m_train\u001b[0;34m(self, corpus)\u001b[0m\n\u001b[1;32m    132\u001b[0m         \u001b[0mcorpus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parse_to_tagged\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tagger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNEChunkParserTagger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_tagged_to_parse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtagged_tokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, train)\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         ClassifierBasedTagger.__init__(\n\u001b[0;32m---> 37\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclassifier_builder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_classifier_builder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m         )\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/tag/sequential.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, feature_detector, train, classifier_builder, classifier, backoff, cutoff_prob, verbose)\u001b[0m\n\u001b[1;32m    637\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    638\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 639\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclassifier_builder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    640\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mchoose_tag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/tag/sequential.py\u001b[0m in \u001b[0;36m_train\u001b[0;34m(self, tagged_corpus, classifier_builder, verbose)\u001b[0m\n\u001b[1;32m    673\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    674\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training classifier ({} instances)\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier_corpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 675\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_classifier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclassifier_builder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier_corpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    676\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    677\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m_classifier_builder\u001b[0;34m(self, train)\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_classifier_builder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         return MaxentClassifier.train(\n\u001b[0;32m---> 42\u001b[0;31m             \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malgorithm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"megam\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgaussian_prior_sigma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m         )\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/classify/maxent.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(cls, train_toks, algorithm, trace, encoding, labels, gaussian_prior_sigma, **cutoffs)\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0malgorithm\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"megam\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m             return train_maxent_classifier_with_megam(\n\u001b[0;32m--> 335\u001b[0;31m                 \u001b[0mtrain_toks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgaussian_prior_sigma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcutoffs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    336\u001b[0m             )\n\u001b[1;32m    337\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0malgorithm\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"tadm\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/classify/maxent.py\u001b[0m in \u001b[0;36mtrain_maxent_classifier_with_megam\u001b[0;34m(train_toks, trace, encoding, labels, gaussian_prior_sigma, **kwargs)\u001b[0m\n\u001b[1;32m   1481\u001b[0m         \u001b[0moptions\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"-multilabel\"\u001b[0m\u001b[0;34m]\u001b[0m  \u001b[0;31m# each possible la\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1482\u001b[0m     \u001b[0moptions\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"multiclass\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainfile_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1483\u001b[0;31m     \u001b[0mstdout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_megam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1484\u001b[0m     \u001b[0;31m# print('./megam_i686.opt ', ' '.join(options))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1485\u001b[0m     \u001b[0;31m# Delete the training file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/classify/megam.py\u001b[0m in \u001b[0;36mcall_megam\u001b[0;34m(args)\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"megam command failed!\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstdout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: megam command failed!"
     ]
    }
   ],
   "source": [
    "# print('10-fold, sentence-level, LegalER coarse-grained, German, nltk NEChunkParser, GIS')\n",
    "# %time cross_validation(10, 'deu', 'conllpos', '../../data/LegalER/dataset_courts/coarse-grained/split', '../../results/14th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10-fold, sentence-document-level, GermEval2014, German, nltk NEChunkParser, megam\n",
      "Cross validation for 31300 samples has begun\n",
      "Training for iteration 1 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-lmzyh_ot']\n",
      "Training for iteration 1 completed\n",
      "Evaluation for iteration 1 started\n",
      "Evaluation for iteration 1 completed\n",
      "Training for iteration 2 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-bcbp5v9j']\n",
      "Training for iteration 2 completed\n",
      "Evaluation for iteration 2 started\n",
      "Evaluation for iteration 2 completed\n",
      "Training for iteration 3 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-tgec57rm']\n",
      "Training for iteration 3 completed\n",
      "Evaluation for iteration 3 started\n",
      "Evaluation for iteration 3 completed\n",
      "Training for iteration 4 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-j1ql82vk']\n"
     ]
    }
   ],
   "source": [
    "# print('10-fold, sentence-document-level, GermEval2014, German, nltk NEChunkParser, GIS')\n",
    "# %time cross_validation(10, 'deu', 'germeval', '../../data/GermEval2014/split', '../../results/15th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for iteration 1\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-1dedqduh']\n",
      "Training for iteration 2\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-am8ryju1']\n",
      "Training for iteration 3\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-yftmgkrt']\n",
      "Training for iteration 4\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-eydgh9d_']\n",
      "Training for iteration 5\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-n229ua_g']\n",
      "Training for iteration 6\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-9ffgtrkt']\n",
      "Training for iteration 7\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-8l5b3frj']\n",
      "Training for iteration 8\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-5ctynx2y']\n",
      "Training for iteration 9\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-clo0jpe8']\n",
      "Training for iteration 10\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-ds12k4h3']\n",
      "results exported to: ../../results/3rd.csv\n"
     ]
    }
   ],
   "source": [
    "# print('10-fold, document-level, Conll2003, English, nltk NEChunkParser, GIS')\n",
    "# %time cross_validation(10, 'eng', 'conll', '../../data/conll2003/spliteng', '../../results/16th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8-fold, document-level, SEC-filings, English, nltk NEChunkParser, IIS\n",
      "Cross validation for 8 samples has begun\n",
      "Training for iteration 1 started\n",
      "The algorithm IIS is being used!\n",
      "  ==> Training (10 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.07944        0.961\n",
      "             2          -0.06365        0.961\n",
      "             3          -0.05480        0.971\n",
      "             4          -0.04446        0.983\n",
      "             5          -0.03736        0.989\n",
      "             6          -0.03240        0.992\n",
      "             7          -0.02876        0.994\n",
      "             8          -0.02596        0.995\n",
      "             9          -0.02372        0.996\n",
      "         Final          -0.02190        0.997\n",
      "Training for iteration 1 completed\n",
      "Evaluation for iteration 1 started\n",
      "Evaluation for iteration 1 completed\n",
      "Training for iteration 2 started\n",
      "The algorithm IIS is being used!\n",
      "  ==> Training (10 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.07944        0.966\n",
      "             2          -0.05604        0.966\n",
      "             3          -0.04944        0.973\n",
      "             4          -0.04052        0.985\n",
      "             5          -0.03430        0.989\n",
      "             6          -0.02990        0.992\n",
      "             7          -0.02663        0.994\n",
      "             8          -0.02410        0.995\n",
      "             9          -0.02206        0.996\n",
      "         Final          -0.02039        0.997\n",
      "Training for iteration 2 completed\n",
      "Evaluation for iteration 2 started\n",
      "Evaluation for iteration 2 completed\n",
      "Training for iteration 3 started\n",
      "The algorithm IIS is being used!\n",
      "  ==> Training (10 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.07944        0.961\n",
      "             2          -0.06374        0.961\n",
      "             3          -0.05528        0.970\n",
      "             4          -0.04522        0.983\n",
      "             5          -0.03820        0.989\n",
      "             6          -0.03325        0.991\n",
      "             7          -0.02957        0.994\n",
      "             8          -0.02672        0.995\n",
      "             9          -0.02444        0.996\n",
      "         Final          -0.02256        0.996\n",
      "Training for iteration 3 completed\n",
      "Evaluation for iteration 3 started\n",
      "Evaluation for iteration 3 completed\n",
      "Training for iteration 4 started\n",
      "The algorithm IIS is being used!\n",
      "  ==> Training (10 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.07944        0.964\n",
      "             2          -0.05924        0.964\n",
      "             3          -0.05232        0.971\n",
      "             4          -0.04322        0.982\n",
      "             5          -0.03677        0.988\n",
      "             6          -0.03216        0.991\n",
      "             7          -0.02870        0.993\n",
      "             8          -0.02601        0.995\n",
      "             9          -0.02383        0.996\n",
      "         Final          -0.02204        0.996\n",
      "Training for iteration 4 completed\n",
      "Evaluation for iteration 4 started\n",
      "Evaluation for iteration 4 completed\n",
      "Training for iteration 5 started\n",
      "The algorithm IIS is being used!\n",
      "  ==> Training (10 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.07944        0.960\n",
      "             2          -0.06779        0.960\n",
      "             3          -0.05942        0.966\n",
      "             4          -0.04851        0.983\n",
      "             5          -0.04091        0.988\n",
      "             6          -0.03554        0.992\n",
      "             7          -0.03156        0.994\n",
      "             8          -0.02848        0.995\n",
      "             9          -0.02602        0.996\n",
      "         Final          -0.02400        0.996\n",
      "Training for iteration 5 completed\n",
      "Evaluation for iteration 5 started\n",
      "Evaluation for iteration 5 completed\n",
      "Training for iteration 6 started\n",
      "The algorithm IIS is being used!\n",
      "  ==> Training (10 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.07944        0.965\n",
      "             2          -0.05807        0.965\n",
      "             3          -0.05111        0.972\n",
      "             4          -0.04223        0.984\n",
      "             5          -0.03598        0.988\n",
      "             6          -0.03151        0.991\n",
      "             7          -0.02816        0.993\n",
      "             8          -0.02555        0.995\n",
      "             9          -0.02344        0.996\n",
      "         Final          -0.02170        0.996\n",
      "Training for iteration 6 completed\n",
      "Evaluation for iteration 6 started\n",
      "Evaluation for iteration 6 completed\n",
      "Training for iteration 7 started\n",
      "The algorithm IIS is being used!\n",
      "  ==> Training (10 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.07944        0.964\n",
      "             2          -0.05894        0.964\n",
      "             3          -0.05148        0.972\n",
      "             4          -0.04214        0.984\n",
      "             5          -0.03561        0.989\n",
      "             6          -0.03101        0.992\n",
      "             7          -0.02759        0.994\n",
      "             8          -0.02495        0.995\n",
      "             9          -0.02283        0.996\n",
      "         Final          -0.02110        0.997\n",
      "Training for iteration 7 completed\n",
      "Evaluation for iteration 7 started\n",
      "Evaluation for iteration 7 completed\n",
      "Training for iteration 8 started\n",
      "The algorithm IIS is being used!\n",
      "  ==> Training (10 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.07944        0.966\n",
      "             2          -0.05693        0.966\n",
      "             3          -0.04963        0.973\n",
      "             4          -0.04049        0.984\n",
      "             5          -0.03409        0.990\n",
      "             6          -0.02959        0.992\n",
      "             7          -0.02626        0.994\n",
      "             8          -0.02370        0.995\n"
     ]
    }
   ],
   "source": [
    "# print('8-fold, document-level, SEC-filings, English, nltk NEChunkParser, GIS')\n",
    "# %time cross_validation(8, 'eng', 'conll', '../../data/SEC-filings/CONLL-format/data/split', '../../results/17th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10-fold, sentence-document-level, WNUT17, English, nltk NEChunkParser, megam\n",
      "Cross validation for 5690 samples has begun\n",
      "Training for iteration 1 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-j2qmpmsm']\n",
      "Training for iteration 1 completed\n",
      "Evaluation for iteration 1 started\n",
      "Evaluation for iteration 1 completed\n",
      "Training for iteration 2 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-1h0yfeg_']\n",
      "Training for iteration 2 completed\n",
      "Evaluation for iteration 2 started\n",
      "Evaluation for iteration 2 completed\n",
      "Training for iteration 3 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-3wfn72s_']\n",
      "Training for iteration 3 completed\n",
      "Evaluation for iteration 3 started\n",
      "Evaluation for iteration 3 completed\n",
      "Training for iteration 4 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-yi3lwr3w']\n",
      "Training for iteration 4 completed\n",
      "Evaluation for iteration 4 started\n",
      "Evaluation for iteration 4 completed\n",
      "Training for iteration 5 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-214tfdlr']\n",
      "Training for iteration 5 completed\n",
      "Evaluation for iteration 5 started\n",
      "Evaluation for iteration 5 completed\n",
      "Training for iteration 6 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-vrbcygys']\n",
      "Training for iteration 6 completed\n",
      "Evaluation for iteration 6 started\n",
      "Evaluation for iteration 6 completed\n",
      "Training for iteration 7 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-60s_mip8']\n",
      "Training for iteration 7 completed\n",
      "Evaluation for iteration 7 started\n",
      "Evaluation for iteration 7 completed\n",
      "Training for iteration 8 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-n1en13uo']\n",
      "Training for iteration 8 completed\n",
      "Evaluation for iteration 8 started\n",
      "Evaluation for iteration 8 completed\n",
      "Training for iteration 9 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-b2kq3zu3']\n",
      "Training for iteration 9 completed\n",
      "Evaluation for iteration 9 started\n",
      "Evaluation for iteration 9 completed\n",
      "Training for iteration 10 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-des2vyiw']\n",
      "Training for iteration 10 completed\n",
      "Evaluation for iteration 10 started\n",
      "Evaluation for iteration 10 completed\n",
      "results exported to: ../../results/6th.csv\n",
      "Cross validation complete.\n"
     ]
    }
   ],
   "source": [
    "# print('10-fold, sentence-document-level, WNUT17, English, nltk NEChunkParser, GIS')\n",
    "# %time cross_validation(10, 'eng', 'conllpos', '../../data/WNUT17/CONLL-format/data/split', '../../results/18th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('10-fold, Conll2003 + GermEval2014, German, nltk NEChunkParser, GIS')\n",
    "%time cross_validation(10, 'ger', 'conll', '../../data/GermEval2014/split/final', '../../results/20th.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}