cross_validation_megam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.corpus.reader import ConllCorpusReader\n",
    "import nltk.chunk.named_entity\n",
    "from nltk.classify import MaxentClassifier\n",
    "import os\n",
    "import fnmatch\n",
    "from random import shuffle\n",
    "\n",
    "nltk.config_megam('../../megam_i686.opt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_classifier_builder(self, train):\n",
    "        print(\"The algorithm megam is being used!\")\n",
    "        return MaxentClassifier.train(\n",
    "            train, algorithm=\"megam\", gaussian_prior_sigma=1, trace=3\n",
    "        )\n",
    "    \n",
    "nltk.chunk.named_entity.NEChunkParserTagger._classifier_builder = new_classifier_builder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement cv myself\n",
    "# inside the loop: training, predict\n",
    "# define scoring function with the previous function which says a label is right or not\n",
    "# k fold -> list of data splits -> chunked_sents \n",
    "# not round number of samples? divide them. three folds with 9 samples instead of one fold with 7\n",
    "\n",
    "# def filecount(dir):\n",
    "#     path, dirs, files = next(os.walk(dir))\n",
    "#     file_count = len(files)\n",
    "#     return file_count\n",
    "\n",
    "def filecount(dir):\n",
    "    num_files = len(fnmatch.filter(os.listdir(dir),'eng.*')) + len(fnmatch.filter(os.listdir(dir),'deu.*'))\n",
    "    return num_files\n",
    "\n",
    "def cross_validation(k, language, dataform, datadir, resultsdir):\n",
    "    if language != 'eng' and language != 'deu': raise ValueError('Language parameter incorrect.')\n",
    "    columnlist = ['IOB Accuracy','Precision:','Recall','F-Measure', 'True Positives', 'False Positives', 'False Negatives']\n",
    "    df = pd.DataFrame(columns= columnlist)\n",
    "    filesum = filecount(datadir)\n",
    "    samplels = []\n",
    "    for i in range(filesum):\n",
    "        with open(datadir + '/' + language + '.' + str(i)) as ele:\n",
    "            samplels.append(ele.read())\n",
    "    samplels = list(filter(None, samplels))\n",
    "    shuffle(samplels)\n",
    "    print('Cross validation for ' + str(len(samplels)) + ' samples has begun')\n",
    "    foldsize = len(samplels) // k\n",
    "    num_extra_bins = len(samplels) % k # the number of extra sized bins\n",
    "    foldsizes = [foldsize + (i < num_extra_bins) for i in range(k)]\n",
    "    trainls = ['' for i in range(k)]\n",
    "    for i in range(k):\n",
    "        for j in range(foldsizes[i]):\n",
    "            trainls[i] += samplels[j+ sum(foldsizes[0:i])] + '\\n'\n",
    "    for i in range(k):\n",
    "        print(\"Training for iteration \" + str(i+1) + ' started')\n",
    "        trainingls = trainls[:i] + trainls[i+1:]\n",
    "        testls = trainls[i]\n",
    "        with open(datadir + '/CV' + '/iteration' + str(i) + '.train', 'w') as tr:\n",
    "            for j in range(len(trainingls)):\n",
    "                tr.write(trainingls[j])\n",
    "        if dataform == 'conllpos':\n",
    "            training = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.train', ['words', 'chunk', 'pos'])\n",
    "        elif language == 'deu' and dataform == 'conll':\n",
    "            training = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.train', ['words', 'ignore', 'pos', 'ignore', 'chunk'])\n",
    "        elif language == 'eng' and dataform == 'conll':\n",
    "            training = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.train', ['words', 'pos', 'ignore', 'chunk'])\n",
    "        elif language == 'deu' and dataform == 'germeval':\n",
    "            training = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.train', ['ignore', 'words', 'chunk', 'ignore', 'pos'])\n",
    "        else: raise ValueError(\"This combination of data language and data form does not exist.\")\n",
    "        ne_chunker = nltk.chunk.named_entity.NEChunkParser(training.chunked_sents())\n",
    "        print(\"Training for iteration \" + str(i+1) + ' completed')\n",
    "\n",
    "        with open(datadir + '/CV' + '/iteration' + str(i) + '.test', 'w') as te:\n",
    "            te.write(testls)\n",
    "        if dataform == 'conllpos':\n",
    "            test = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.test', ['words', 'chunk', 'pos'])\n",
    "        elif language == 'deu' and dataform == 'conll':\n",
    "            test = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.test', ['words', 'ignore', 'pos', 'ignore', 'chunk'])\n",
    "        elif language == 'eng' and dataform == 'conll':\n",
    "            test = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.test', ['words', 'pos', 'ignore', 'chunk'])\n",
    "        elif language == 'deu' and dataform == 'germeval':\n",
    "            test = ConllCorpusReader(datadir + '/CV', 'iteration' + str(i) + '.test', ['ignore', 'words', 'chunk', 'ignore', 'pos'])\n",
    "        else: raise ValueError(\"This combination of data language and data form does not exist.\")\n",
    "        print(\"Evaluation for iteration \" + str(i+1) + ' started')\n",
    "        results = ne_chunker.evaluate(test.chunked_sents())\n",
    "        bufferdf = pd.DataFrame([[results.accuracy(),results.precision(),results.recall(),results.f_measure(),results._tp_num, results._fp_num, results._fn_num]], columns = columnlist, index = ['Iteration' + str(i + 1)])\n",
    "        df = df.append(bufferdf)\n",
    "        print(\"Evaluation for iteration \" + str(i+1) + ' completed')\n",
    "    df.to_csv(resultsdir)\n",
    "    print('results exported to: ' + resultsdir)\n",
    "    print('Cross validation complete.')\n",
    "\n",
    "        #comments:\n",
    "        #read the files in a list \n",
    "        #put the texts to each fold together\n",
    "        #train\n",
    "        #predict, use the previous scoring function\n",
    "        #export\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for iteration 1\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-1pdvrsdc']\n",
      "Training for iteration 2\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-exri4mkp']\n",
      "Training for iteration 3\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-9exchqnj']\n",
      "Training for iteration 4\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-9ybh6fka']\n",
      "Training for iteration 5\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-atskr0eb']\n",
      "Training for iteration 6\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-ayf391a2']\n",
      "Training for iteration 7\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-6_volivx']\n",
      "Training for iteration 8\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-yghwg0g0']\n",
      "Training for iteration 9\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-1ou2ft7b']\n",
      "Training for iteration 10\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-cjrtwgtv']\n",
      "results exported to: ../../results/2nd.csv\n"
     ]
    }
   ],
   "source": [
    "# finished\n",
    "# print('10-fold, document-level, Conll2003, German, nltk NEChunkParser, megam')\n",
    "# %time cross_validation(10, 'deu', 'conll', '../../data/conll2003/splitdeu', '../../results/1st.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10-fold, sentence-level, LegalER coarse-grained, German, nltk NEChunkParser, megam\n",
      "Cross validation for 66723 samples has begun\n",
      "Training for iteration 1 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-in_z6_78']\n",
      "\n",
      "None\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "megam command failed!",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-209b91047a61>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'10-fold, sentence-level, LegalER coarse-grained, German, nltk NEChunkParser, megam'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mcross_validation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'deu'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'conllpos'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'../../data/LegalER/dataset_courts/coarse-grained/split'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'../../results/2nd.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-2-e3468eea4615>\u001b[0m in \u001b[0;36mcross_validation\u001b[0;34m(k, language, dataform, datadir, resultsdir)\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0mtraining\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mConllCorpusReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatadir\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/CV'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'iteration'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'.train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'words'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'chunk'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'pos'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"This combination of data language and data form does not exist.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mne_chunker\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_entity\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNEChunkParser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunked_sents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training for iteration \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' completed'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, train)\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m_train\u001b[0;34m(self, corpus)\u001b[0m\n\u001b[1;32m    132\u001b[0m         \u001b[0mcorpus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parse_to_tagged\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tagger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNEChunkParserTagger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_tagged_to_parse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtagged_tokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, train)\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         ClassifierBasedTagger.__init__(\n\u001b[0;32m---> 37\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclassifier_builder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_classifier_builder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m         )\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/tag/sequential.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, feature_detector, train, classifier_builder, classifier, backoff, cutoff_prob, verbose)\u001b[0m\n\u001b[1;32m    637\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    638\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 639\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclassifier_builder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    640\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mchoose_tag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/tag/sequential.py\u001b[0m in \u001b[0;36m_train\u001b[0;34m(self, tagged_corpus, classifier_builder, verbose)\u001b[0m\n\u001b[1;32m    673\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    674\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training classifier ({} instances)\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier_corpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 675\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_classifier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclassifier_builder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier_corpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    676\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    677\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m_classifier_builder\u001b[0;34m(self, train)\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_classifier_builder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         return MaxentClassifier.train(\n\u001b[0;32m---> 42\u001b[0;31m             \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malgorithm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"megam\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgaussian_prior_sigma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m         )\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/classify/maxent.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(cls, train_toks, algorithm, trace, encoding, labels, gaussian_prior_sigma, **cutoffs)\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0malgorithm\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"megam\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m             return train_maxent_classifier_with_megam(\n\u001b[0;32m--> 335\u001b[0;31m                 \u001b[0mtrain_toks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgaussian_prior_sigma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcutoffs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    336\u001b[0m             )\n\u001b[1;32m    337\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0malgorithm\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"tadm\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/classify/maxent.py\u001b[0m in \u001b[0;36mtrain_maxent_classifier_with_megam\u001b[0;34m(train_toks, trace, encoding, labels, gaussian_prior_sigma, **kwargs)\u001b[0m\n\u001b[1;32m   1481\u001b[0m         \u001b[0moptions\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"-multilabel\"\u001b[0m\u001b[0;34m]\u001b[0m  \u001b[0;31m# each possible la\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1482\u001b[0m     \u001b[0moptions\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"multiclass\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainfile_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1483\u001b[0;31m     \u001b[0mstdout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_megam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1484\u001b[0m     \u001b[0;31m# print('./megam_i686.opt ', ' '.join(options))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1485\u001b[0m     \u001b[0;31m# Delete the training file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/classify/megam.py\u001b[0m in \u001b[0;36mcall_megam\u001b[0;34m(args)\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"megam command failed!\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstdout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: megam command failed!"
     ]
    }
   ],
   "source": [
    "# training data file too big(24M), biggest so far that works(9M).\n",
    "# print('10-fold, sentence-level, LegalER coarse-grained, German, nltk NEChunkParser, megam')\n",
    "# %time cross_validation(10, 'deu', 'conllpos', '../../data/LegalER/dataset_courts/coarse-grained/split', '../../results/2nd.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10-fold, sentence-document-level, GermEval2014, German, nltk NEChunkParser, megam\n",
      "Cross validation for 31300 samples has begun\n",
      "Training for iteration 1 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-lmzyh_ot']\n",
      "Training for iteration 1 completed\n",
      "Evaluation for iteration 1 started\n",
      "Evaluation for iteration 1 completed\n",
      "Training for iteration 2 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-bcbp5v9j']\n",
      "Training for iteration 2 completed\n",
      "Evaluation for iteration 2 started\n",
      "Evaluation for iteration 2 completed\n",
      "Training for iteration 3 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-tgec57rm']\n",
      "Training for iteration 3 completed\n",
      "Evaluation for iteration 3 started\n",
      "Evaluation for iteration 3 completed\n",
      "Training for iteration 4 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-j1ql82vk']\n",
      "Training for iteration 4 completed\n",
      "Evaluation for iteration 4 started\n",
      "Evaluation for iteration 4 completed\n",
      "Training for iteration 5 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-6byqnecu']\n",
      "Training for iteration 5 completed\n",
      "Evaluation for iteration 5 started\n",
      "Evaluation for iteration 5 completed\n",
      "Training for iteration 6 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-id2c5vcr']\n",
      "Training for iteration 6 completed\n",
      "Evaluation for iteration 6 started\n",
      "Evaluation for iteration 6 completed\n",
      "Training for iteration 7 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-69oht01r']\n",
      "Training for iteration 7 completed\n",
      "Evaluation for iteration 7 started\n",
      "Evaluation for iteration 7 completed\n",
      "Training for iteration 8 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-8eyt0yuw']\n",
      "Training for iteration 8 completed\n",
      "Evaluation for iteration 8 started\n",
      "Evaluation for iteration 8 completed\n",
      "Training for iteration 9 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-hf1kmyss']\n",
      "Training for iteration 9 completed\n",
      "Evaluation for iteration 9 started\n",
      "Evaluation for iteration 9 completed\n",
      "Training for iteration 10 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-t8djx5rx']\n",
      "Training for iteration 10 completed\n",
      "Evaluation for iteration 10 started\n",
      "Evaluation for iteration 10 completed\n",
      "results exported to: ../../results/3rd.csv\n",
      "Cross validation complete.\n"
     ]
    }
   ],
   "source": [
    "# finished\n",
    "# print('10-fold, sentence-document-level, GermEval2014, German, nltk NEChunkParser, megam')\n",
    "# %time cross_validation(10, 'deu', 'germeval', '../../data/GermEval2014/split', '../../results/3rd.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for iteration 1\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-1dedqduh']\n",
      "Training for iteration 2\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-am8ryju1']\n",
      "Training for iteration 3\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-yftmgkrt']\n",
      "Training for iteration 4\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-eydgh9d_']\n",
      "Training for iteration 5\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-n229ua_g']\n",
      "Training for iteration 6\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-9ffgtrkt']\n",
      "Training for iteration 7\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-8l5b3frj']\n",
      "Training for iteration 8\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-5ctynx2y']\n",
      "Training for iteration 9\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-clo0jpe8']\n",
      "Training for iteration 10\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-ds12k4h3']\n",
      "results exported to: ../../results/3rd.csv\n"
     ]
    }
   ],
   "source": [
    "# finished\n",
    "# print('10-fold, document-level, Conll2003, English, nltk NEChunkParser, megam')\n",
    "# %time cross_validation(10, 'eng', 'conll', '../../data/conll2003/spliteng', '../../results/4th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8-fold, document-level, SEC-filings, English, nltk NEChunkParser, megam\n",
      "Cross validation for 8 samples has begun\n",
      "Training for iteration 1 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-bg5bq1kp']\n",
      "Training for iteration 1 completed\n",
      "Evaluation for iteration 1 started\n",
      "Evaluation for iteration 1 completed\n",
      "Training for iteration 2 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-y83bg3rn']\n",
      "Training for iteration 2 completed\n",
      "Evaluation for iteration 2 started\n",
      "Evaluation for iteration 2 completed\n",
      "Training for iteration 3 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-oxpzr5hn']\n",
      "Training for iteration 3 completed\n",
      "Evaluation for iteration 3 started\n",
      "Evaluation for iteration 3 completed\n",
      "Training for iteration 4 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-2t6_u2eu']\n",
      "Training for iteration 4 completed\n",
      "Evaluation for iteration 4 started\n",
      "Evaluation for iteration 4 completed\n",
      "Training for iteration 5 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-ryies8_x']\n",
      "Training for iteration 5 completed\n",
      "Evaluation for iteration 5 started\n",
      "Evaluation for iteration 5 completed\n",
      "Training for iteration 6 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-2ztcahqn']\n",
      "Training for iteration 6 completed\n",
      "Evaluation for iteration 6 started\n",
      "Evaluation for iteration 6 completed\n",
      "Training for iteration 7 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-n3qxn0n0']\n",
      "Training for iteration 7 completed\n",
      "Evaluation for iteration 7 started\n",
      "Evaluation for iteration 7 completed\n",
      "Training for iteration 8 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-pb7_q430']\n",
      "Training for iteration 8 completed\n",
      "Evaluation for iteration 8 started\n",
      "Evaluation for iteration 8 completed\n",
      "results exported to: ../../results/3rd.csv\n",
      "Cross validation complete.\n"
     ]
    }
   ],
   "source": [
    "# finished\n",
    "# print('8-fold, document-level, SEC-filings, English, nltk NEChunkParser, megam')\n",
    "# %time cross_validation(8, 'eng', 'conll', '../../data/SEC-filings/CONLL-format/data/split', '../../results/5th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10-fold, sentence-document-level, WNUT17, English, nltk NEChunkParser, megam\n",
      "Cross validation for 5690 samples has begun\n",
      "Training for iteration 1 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-j2qmpmsm']\n",
      "Training for iteration 1 completed\n",
      "Evaluation for iteration 1 started\n",
      "Evaluation for iteration 1 completed\n",
      "Training for iteration 2 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-1h0yfeg_']\n",
      "Training for iteration 2 completed\n",
      "Evaluation for iteration 2 started\n",
      "Evaluation for iteration 2 completed\n",
      "Training for iteration 3 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-3wfn72s_']\n",
      "Training for iteration 3 completed\n",
      "Evaluation for iteration 3 started\n",
      "Evaluation for iteration 3 completed\n",
      "Training for iteration 4 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-yi3lwr3w']\n",
      "Training for iteration 4 completed\n",
      "Evaluation for iteration 4 started\n",
      "Evaluation for iteration 4 completed\n",
      "Training for iteration 5 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-214tfdlr']\n",
      "Training for iteration 5 completed\n",
      "Evaluation for iteration 5 started\n",
      "Evaluation for iteration 5 completed\n",
      "Training for iteration 6 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-vrbcygys']\n",
      "Training for iteration 6 completed\n",
      "Evaluation for iteration 6 started\n",
      "Evaluation for iteration 6 completed\n",
      "Training for iteration 7 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-60s_mip8']\n",
      "Training for iteration 7 completed\n",
      "Evaluation for iteration 7 started\n",
      "Evaluation for iteration 7 completed\n",
      "Training for iteration 8 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-n1en13uo']\n",
      "Training for iteration 8 completed\n",
      "Evaluation for iteration 8 started\n",
      "Evaluation for iteration 8 completed\n",
      "Training for iteration 9 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-b2kq3zu3']\n",
      "Training for iteration 9 completed\n",
      "Evaluation for iteration 9 started\n",
      "Evaluation for iteration 9 completed\n",
      "Training for iteration 10 started\n",
      "['../../megam_i686.opt', '-nobias', '-repeat', '10', '-explicit', '-lambda', '1.00', '-tune', '-quiet', 'multiclass', '/tmp/nltk-des2vyiw']\n",
      "Training for iteration 10 completed\n",
      "Evaluation for iteration 10 started\n",
      "Evaluation for iteration 10 completed\n",
      "results exported to: ../../results/6th.csv\n",
      "Cross validation complete.\n"
     ]
    }
   ],
   "source": [
    "# finished\n",
    "# print('10-fold, sentence-document-level, WNUT17, English, nltk NEChunkParser, megam')\n",
    "# %time cross_validation(10, 'eng', 'conllpos', '../../data/WNUT17/CONLL-format/data/split', '../../results/6th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10-fold, Conll2003 + GermEval2014, German, nltk NEChunkParser, megam\n",
      "Cross validation for 32057 samples has begun\n",
      "Training for iteration 1 started\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Inconsistent number of columns:\nBei 1 UNK O.1 O\nGeld 2 UNK O O\nsind 3 UNK O O\nder 4 UNK O O\nProduktivitätswert 5 UNK O O\n 6 UNK O O\nund 7 UNK O O\ndie 8 UNK O O\nDurchhaltekosten 9 UNK O O\n( 10 UNK O O\nfür 11 UNK O O\nAufbewahrung 12 UNK O O\nund 13 UNK O O\nSicherheit 14 UNK O O\n) 15 UNK O O\ngering 16 UNK O O\n, 17 UNK O O\ndie 18 UNK O O\nLiquiditätsprämie 19 UNK O O\njedoch 20 UNK O O\nbedeutend 21 UNK O O\n( 22 UNK O O\n„ 23 UNK O O\nman 24 UNK O O\nkann 25 UNK O O\nsich 26 UNK O O\nwas 27 UNK O O\ndafür 28 UNK O O\nkaufen 29 UNK O O\n“ 30 UNK O O\n) 31 UNK O O\n. 32 UNK O O",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m----------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m   Traceback (most recent call last)",
      "\u001b[0;32m<timed eval>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-e3468eea4615>\u001b[0m in \u001b[0;36mcross_validation\u001b[0;34m(k, language, dataform, datadir, resultsdir)\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0mtraining\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mConllCorpusReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatadir\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/CV'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'iteration'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'.train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'words'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'chunk'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'pos'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"This combination of data language and data form does not exist.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mne_chunker\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_entity\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNEChunkParser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunked_sents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training for iteration \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' completed'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, train)\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m_train\u001b[0;34m(self, corpus)\u001b[0m\n\u001b[1;32m    130\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m         \u001b[0;31m# Convert to tagged sequence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m         \u001b[0mcorpus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parse_to_tagged\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tagger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNEChunkParserTagger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    130\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m         \u001b[0;31m# Convert to tagged sequence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m         \u001b[0mcorpus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parse_to_tagged\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tagger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNEChunkParserTagger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/collections.py\u001b[0m in \u001b[0;36miterate_from\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    404\u001b[0m         \u001b[0;31m# Special case: one lazy sublist\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    405\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lists\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_all_lazy\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 406\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lists\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterate_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    407\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/corpus/reader/util.py\u001b[0m in \u001b[0;36miterate_from\u001b[0;34m(self, start_tok)\u001b[0m\n\u001b[1;32m    302\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current_toknum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtoknum\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current_blocknum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mblock_index\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m             \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stream\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m             assert isinstance(tokens, (tuple, list, AbstractLazySequence)), (\n\u001b[1;32m    306\u001b[0m                 \u001b[0;34m\"block reader %s() should return list or tuple.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/nltk/corpus/reader/conll.py\u001b[0m in \u001b[0;36m_read_grid_block\u001b[0;34m(self, stream)\u001b[0m\n\u001b[1;32m    241\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgrid\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 243\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Inconsistent number of columns:\\n%s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mblock\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    244\u001b[0m             \u001b[0mgrids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    245\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mgrids\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Inconsistent number of columns:\nBei 1 UNK O.1 O\nGeld 2 UNK O O\nsind 3 UNK O O\nder 4 UNK O O\nProduktivitätswert 5 UNK O O\n 6 UNK O O\nund 7 UNK O O\ndie 8 UNK O O\nDurchhaltekosten 9 UNK O O\n( 10 UNK O O\nfür 11 UNK O O\nAufbewahrung 12 UNK O O\nund 13 UNK O O\nSicherheit 14 UNK O O\n) 15 UNK O O\ngering 16 UNK O O\n, 17 UNK O O\ndie 18 UNK O O\nLiquiditätsprämie 19 UNK O O\njedoch 20 UNK O O\nbedeutend 21 UNK O O\n( 22 UNK O O\n„ 23 UNK O O\nman 24 UNK O O\nkann 25 UNK O O\nsich 26 UNK O O\nwas 27 UNK O O\ndafür 28 UNK O O\nkaufen 29 UNK O O\n“ 30 UNK O O\n) 31 UNK O O\n. 32 UNK O O"
     ]
    }
   ],
   "source": [
    "print('10-fold, Conll2003 + GermEval2014, German, nltk NEChunkParser, megam')\n",
    "%time cross_validation(10, 'deu', 'conll', '../../data/GermEval2014/split/final', '../../results/19th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}