word_counter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json # To read JSON files\n",
    "import os # To access files\n",
    "from collections import Counter # Used to count the most occuring words in a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "WORDS_TO_COUNT = [ # The occurences of the words in this list are counted for every file\n",
    "    'cyber',\n",
    "    'security',\n",
    "    'cyber security',\n",
    "    'cybersecurity',\n",
    "    'ddos',\n",
    "    'hacker',\n",
    "    'cybercrime',\n",
    "    'digital',\n",
    "    'protection',\n",
    "    'cloud',\n",
    "    'surveillance systems',\n",
    "    'privacy',\n",
    "    'internet',\n",
    "    'internet of things',\n",
    "    'digitalisation',\n",
    "    'big data',\n",
    "    'digital technologies',\n",
    "    'digital age',\n",
    "    'ict',\n",
    "    'cyber attack',\n",
    "    'egovernment',\n",
    "    'data protection',\n",
    "    'personal data',\n",
    "    'information security',\n",
    "    'financial',\n",
    "]\n",
    "\n",
    "EXCLUDE_LIST = [ # These words are excluded when counting the most occuring words in a file\n",
    "    'and',\n",
    "    'the',\n",
    "    'of',\n",
    "    'to',\n",
    "    'for',\n",
    "    'in',\n",
    "    'a',\n",
    "    'on',\n",
    "    'that',\n",
    "    'be',\n",
    "    'as',\n",
    "    'or',\n",
    "    'not',\n",
    "]\n",
    "\n",
    "counter = 0\n",
    "\n",
    "base_dir = os.path.abspath(os.curdir) + '/files/'\n",
    "\n",
    "print(\"Starting process\")\n",
    "\n",
    "# Loop through all files in the base_dir\n",
    "for filename in os.listdir(base_dir):\n",
    "    \n",
    "    # Skip directories\n",
    "    if os.path.isdir(os.path.join(base_dir, filename)):\n",
    "        continue\n",
    "    \n",
    "    # Open every file\n",
    "    with open(base_dir + filename, 'r') as file:\n",
    "        try:\n",
    "            file_data = json.loads(file.read())\n",
    "        except Exception:\n",
    "            print(\"Formatting '{}' went wrong!\".format(filename))\n",
    "            continue\n",
    "        \n",
    "        # Calculate the words that occur the most\n",
    "        most_occur = Counter(word for word in file_data['content'].split() if word not in EXCLUDE_LIST).most_common(10)\n",
    "        \n",
    "        # Add the new information to the file_data\n",
    "        file_data.update({\n",
    "            'word_counts': {},\n",
    "            'most_common_words': dict(most_occur),\n",
    "        })\n",
    "    \n",
    "        # Count the words that have been specifically defined in WORDS_TO_COUNT\n",
    "        for word in WORDS_TO_COUNT:\n",
    "            file_data['word_counts'][word] = file_data['content'].count(word.lower())\n",
    "        \n",
    "        counter += 1\n",
    "        \n",
    "    with open(base_dir + filename, 'w') as file:\n",
    "        file.write(json.dumps(file_data))\n",
    "        \n",
    "print(\"Done! Processed files: \" + str(counter))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}