Regex_Back_up.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca32f989-e213-4451-b7af-72b6b60c9d1b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import collections\n",
    "import tqdm\n",
    "import itertools\n",
    "import more_itertools\n",
    "import tabulate\n",
    "from tqdm.contrib.concurrent import process_map, thread_map\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from collections import Counter\n",
    "from collections import Counter\n",
    "from nltk.util import ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65c4cca3-dbb3-40e5-be82-b7e4bf79fd8f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"/global/u2/m/masare/ticket-text-analysis/ticket_cleaned.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf78c146-822f-402b-8ae2-c0db9d7febe4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cf0d6af-3b72-4911-965f-3995d5961851",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "patterns = {\n",
    "    \"python\": r\"(?:python|python3|\\S+\\.py)\", # python, analyze separately\n",
    "    \"GPD\": r\"(?:kaon_.part_smearing_opt|pion(?:\\S+)?|b_minus_Dx)\\.exe\", # m4240 Lattice QCD code\n",
    "    \"EXCESS\": r\"exess\", # Dr. Giuseppe Barca from The Australian National University (ANU) to port his existing EXtreme-scale Electronic Structure System (EXESS\n",
    "    \"WEST\": r\"wfreq\\.x\", \n",
    "    \"vasp\": r\"vasp\",\n",
    "    \"CP2k\": R\"cp2k(?:\\S+)?\", \n",
    "    \"QE\": r\"(?:pw|pp|pw2bgw|epw|neb|matdyn|dynmat|dos|pw2wannier90|q2r|ph|turbo_magnon|kgrid|bands|projwfc)\\.x\", # TODO: add all the other QE binaries\n",
    "    \"LAMMPS\": r\"lmp(?:-\\S+|_\\S+)?\",\n",
    "    \"NWCHEM\": r\"nwchem(?:\\S+)?\",\n",
    "    \"JDFTx\": r\"jdftx(?:_\\S+)?\",\n",
    "    \"IOR\": r\"(?:\\/|\\b|_)ior(?:\\b|_)\",\n",
    "    \"OpenMPI\": r\"(?:prted|orted)\",\n",
    "    \"chroma\": r\"(?:cf_)?chroma(?:\\S+)?\",\n",
    "    \"Wannier90\": r\"(?:postw90|wannier90)\\.x\",\n",
    "    \"E3SM\": r\"e3sm\\.exe\",\n",
    "    \"MILC\": r\"(?:ks_spectrum_hisq|su3_rhmc_hisq|gradientFlow|wilson_flow)(?:\\S+)?\", # gradientFlow, ks_spectrum_hisq\n",
    "    \"QLUA\": r\"qlua\", # https://usqcd-software.github.io/QLUA.html\n",
    "    \"Gromacs\": r\"gmx\", \n",
    "    \"BerkeleyGW\": r\"(?:hdf2wfn|kernel|absorption|epsilon|summarize_eigenvectors|siesta2bgw|sigma|eps0sym|epsinvomega|nonlinearoptics|inteqp|parabands|plotxct|siesta2bgw|wrapper_epsilon)\\.(?:cplx|real)?\\.x(?:\\S+)?\",\n",
    "    \"QMCPACK\": r\"qmcpack\",\n",
    "    \"OmniFold\": r\"omnifold\", # https://github.com/hep-lbdl/OmniFold\n",
    "    \"ONTEP\" : r\"onetep\",\n",
    "    \"WRF\": r\"wrf(?:\\.|_)(?:\\S+)?(?:exe|py|io|wy2017)(?:\\S+)?\",\n",
    "    \"COAWST\": r\"coawstM\", # https://www.usgs.gov/centers/whcmsc/science/coawst-a-coupled-ocean-atmosphere-wave-sediment-transport-modeling-system\n",
    "    \"M3GNet\": r\"M3GNet\", # \"https://github.com/materialsvirtuallab/m3gnet\n",
    "    \"LALIBE\": r\"lalibe\", # https://github.com/callat-qcd/lalibe\n",
    "    \"nvidia-smi\": r\"nvidia-smi\",\n",
    "    \"Octo-Tiger\": r\"octotiger\", # https://octotiger.stellar-group.org\n",
    "    \"GWU-QCD\": r\"(?:overlap_curseq|overlap_inverter|2pt_cfun_calc_milc2kentucky|wilsoninverter)\", # https://arxiv.org/abs/2001.05706, m102\n",
    "    \"CPS\": r\"(?:staggered_\\S+|wilsoninverter(?:\\S+)CPS)\", # Columbia Physics System Lattice QCD, intersection with GWU?\n",
    "    \"XGC\": r\"xgc(?:20230618|_|-es|-eem|-s|-gpu|-no_gpu|a-cpp)(?:\\S+)?\", # XGC (incl. xgca) https://epsi.pppl.gov/computing/xgc-1\n",
    "    \"XGCm\": r\"XGCm\", # https://xgcm.readthedocs.io/en/latest/index.html\n",
    "    \"halld_recon\": r\"halld_recon\", # JLab GLUEx workflow engine swif https://halldweb.jlab.org/DocDB/0037/003793/002/NERSC_AY2018_Hall_D.pdf\n",
    "    \"OSIRIS\": r\"osiris\", # https://picksc.physics.ucla.edu/osiris.html\n",
    "    #\"GENE\": r\"\\sgene(?:|_\\S+)\", #pattern matched as GENE instead of generates #added b and \\b for the white space but ended up getting errors\n",
    "    \"shifter\": r\"shifter\", # containers (shifter)\n",
    "    \"podman\": r\"podman-hpc\", # containers (podman-hpc)\n",
    "    \"flux\": r\"flux\", # flux workflow manager\n",
    "    \"Athena++\": r\"athena\", # maybe nocr_athhena doesn't belong? # https://www.athena-astro.app\n",
    "    \"WarpX\": r\"warpx\",\n",
    "    \"HPL\": r\"xhpl\",\n",
    "    \"AToM-2 SciDAC\": r\"(?:xolotl|run_parallel\\.exe)\", # https://github.com/ORNL-Fusion/xolotl, run_parallel also used in m3195, \n",
    "    \"EPOCH\": r\"epoch3d\", # (Extendable PIC Open Collaboration)\n",
    "    \"CGYRO\": r\"cgyro\", # https://gafusion.github.io/doc/cgyro.html, OpenACC Fortran\n",
    "    \"DFT-FE\": r\"dftfe\", # https://sites.google.com/umich.edu/dftfe\n",
    "    \"M3D-C1\": r\"m3dc1(?:\\S+)?\", # https://m3dc1.pppl.gov\n",
    "    \"Tristan\": r\"tristan\", # https://princetonuniversity.github.io/tristan-v2/\n",
    "    \"Fornax\": r\"fornax\", # https://www.astro.princeton.edu/~burrows/scidac4/fornax.html\n",
    "    \"ChronusQ\": r\"chronusq\", # https://arxiv.org/abs/1905.01381, https://github.com/xsligroup/chronusq_public MPI C++ OpenMP (gpu?)\n",
    "    \"VORPAL\": r\"vorpal\",  # 10.1016/j.jcp.2003.11.004 C++\n",
    "    \"ACE3P\": r\"track3p\", # https://abpcomputing.web.cern.ch/codes/codes_pages/ACE3P/\n",
    "    \"QDP++\": r\"(?:qdpxx|DStill)\", # /global/cfs/cdirs/m2986/software/perlmutter/code-qdpxx-6-2-23-gpu/install/wit/bin/DStill\"\n",
    "    \"FLASH\": r\"flash\", # https://flash.rochester.edu/site/flashcode/\n",
    "    \"MCFM\": r\"mcfm\", # \"https://mcfm.fnal.gov\"\n",
    "    \"TaskFarmer\": r\"tf_worker\", # TaskFarmer workflow manager\n",
    "    \"VLASIATOR\": r\"vlasiator\", # https://www.helsinki.fi/en/researchgroups/vlasiator\n",
    "    \"PROTEUS\": r\"proteus\", # TODO: but which one? https://github.com/nasa/PROTEUS or https://github.com/erdc/proteus or ???\n",
    "    \"SIESTA\": r\"siesta\", # mat sci code\n",
    "    \"SW4\": r\"sw4\", # Seismic\n",
    "    \"Gkeyll\": r\"gkyl\", #  https://github.com/ammarhakim/gkyl C++ CUDA NCCL\n",
    "    \"Trillinos\": r\"Tpetra\",\n",
    "    \"Petra-M\": r\"petram(?:s|p)\", # /global/common/software/petram/20230724/src/PetraM_Driver/README.md\n",
    "    \"HACC\": r\"hacc\",\n",
    "    \"CESM\": r\"cesm\",\n",
    "    \"Maestro\": r\"Maestro\",\n",
    "    \"ATS\": r\"(?:\\/|\\b)ats\",  # TODO: review this pattern... as high potential for false positives\n",
    "    \"NIMROD\": r\"nimrod\",\n",
    "    \"QBOX\": r\"qbox\",\n",
    "    \"ChomboCrunch\": r\"crunchDriver\",\n",
    "    \"FHI-aims\": r\"aims\\.x\",\n",
    "    \"phoenix\": r\"phoenix\",\n",
    "    \"MFDn\": r\"xmfdn\",  # xmfdn-vnn-lan , vnn = 2d body, lan = lanczos (lobpcg), xmfdn-(h2|vnn)-(lan|lobpcg)\n",
    "    \"Julia\": r\"(?:julia|\\S+\\.jl)\",\n",
    "    \"SOLPS\": r\"b2mn\\.exe\",\n",
    "    \"MetaHipMer\": r\"mhm2\",\n",
    "    \"CQL3d\": r\"xcql3d\",\n",
    "    \"NCCL Benchmark\": r\"all_reduce_perf\",\n",
    "    \"Fireworks\": r\"(?:continuous|repacking)_rlaunch(?:\\S+)?\\.sh\", # Fireworks workflow tool, Materials Genome among others, repacking_rlaunch_perlmutter.sh\n",
    "    \"DFTB+\": r\"dftb\\+\", \n",
    "    \"Hadrons\": r\"HadronsXmlRun(?:\\S+)?\", # https://aportelli.github.io/Hadrons-doc/#/ Grid based Lattice QCD workflows\n",
    "    \"AMReX\": r\"main3d\\.\", # name.compiler.arch.type.opt.gpu.ex ?\n",
    "    \"stochasticGW\": r\"sgw_perl\\.x\",\n",
    "    \"PeleC\": r\"PeleC\",\n",
    "    \"Octopus\": r\"octopus\",\n",
    "    \"GTC\": r\"gtc\",  # TODO: review this pattern... as high potential for false positives\n",
    "    \"GTS\": r\"gts\",  # TODO: review this pattern... as high potential for false positives\n",
    "    \"NVIDIA Tools\": r\"(?:nsys|ncu|nvprof)\", # Nsight systems, compute\n",
    "    \"CUDA-gdb\": r\"cuda-gdb\",\n",
    "    \"GAMESS\": r\"gamess\",\n",
    "    \"DASK\": \"dask-worker\", # fold into \"python\" ?\n",
    "    \"OSU Benchmarks\": r\"osu_\\S+\",\n",
    "    \"PARATEC\": r\"paratec\",\n",
    "    \"valgrind\": r\"valgrind\",\n",
    "    \"Berkeley-CS267\": r\"kmer_hash_(?:19|51)\", #https://github.com/Berkeley-CS267/hw3kmer_hash_51\n",
    "    \"gdb\": r\"gdb\",\n",
    "    \"Check MPI\": r\"check-(?:mpi|hybrid)\\.(?:\\S+)\\.pm\", # NERSC provided\n",
    "    \"ARM Forge\": r\"forge-backend\", # ARM Forge debugger\n",
    "    \"MATLAB\": r\"matlab\",\n",
    "    \"ABINIT\": r\"abinit\",\n",
    "    \"YAMBO\": r\"yambo\",\n",
    "    \"PolyFTS\": r\"PolyFTS\",\n",
    "    \"STREAM\": r\"stream\",\n",
    "    \"WEST\": r\"w(?:stat|freq|bse)\\.x\",\n",
    "    \"NCO\": r\"\\snc(?:remap|ks|diff|bo|es|rcat|add|substract|multiply|divide|pack|unpack)\", # https://nco.sourceforge.net/nco.html, look into this pattern well\n",
    "    \"SuperLU\": r\"superlu_dist\",\n",
    "    \"GEMM\": r\"gemm\", # DGEMM benchmark\n",
    "    \"ALICE\": r\"(?:process_LHC|process_embed|process_mc|alicepro|\\/alice\\/)\", # ALICE experiment data processing process_LHC18f5b.sh\n",
    "    \"MOLPRO\": r\"molpro\",\n",
    "    #\"DESI\": r\"desi\",\n",
    "    \"WRF\": r\"wrf\",\n",
    "    \"E3SM\": r\"E3SM\",\n",
    "    \"Alphafold\": \"af2c\", # /global/u1/m/mugao/projects/af2complex/git/src\n",
    "    \"Fungal/Algal Genome annotation\": r\"fnglanot\",\n",
    "    \"TOAST\": r\"toast\",\n",
    "    \"GX\": r\"gx\", # nmandell, https://gx.rtfd.io\n",
    "    \"KBASE\": r\"kbase\",\n",
    "    \"QTools\": r\"qtools\",\n",
    "    \"PyZFS\": r\"pyzfs\", #     https://github.com/hema-ted/pyzfs\n",
    "    \"FHI-aims\": r\"fhi(?:\\S+)aims\",\n",
    "    \"Chombo\": r\"(?:\\S+)\\.Linux\\.64\\.(?:g\\+\\+|CC)\\.ftn\\.(?:\\S+)\\.ex\", # e.g. driver2d.Linux.64.g++.ftn.OPT.MPI.PETSC.ex\n",
    "    \"PFLOTRAN\": r\"pflotran\",\n",
    "    \"NRQCD\": r\"NRQCD\", # /global/cfs/cdirs/m3760/Wei-Ping/measure/NRQCD\n",
    "}\n",
    "\n",
    "compiled_patterns = {\n",
    "    name: re.compile(pattern, re.IGNORECASE)\n",
    "    for name, pattern in patterns.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99021197-4ba2-417d-afb7-90835d9f3c75",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def match(string, patterns):\n",
    "    applications = []\n",
    "    for name, pattern in patterns.items():\n",
    "        #print(name)\n",
    "        #print(applications)\n",
    "        if pattern.search(string):\n",
    "            #print(\"found\", name)\n",
    "            applications.append(name) \n",
    "    return applications\n",
    "\n",
    "def f(x):\n",
    "    return match(x, compiled_patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bfd86b1-2bea-4c40-8885-221ebf0115cd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "names = process_map(f, data['text_data'], max_workers=32, chunksize=1000) #play with the numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fea459d5-6ff6-435c-b638-ed8e166c5328",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.insert(2,\"Applications\",names,True)\n",
    "data['text_data'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ed4f87b-1fb4-4128-9041-40cc7a068fc5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "match(data['text_data'][0], {\"k\": re.compile('Lisa', re.IGNORECASE), \"p\": re.compile('thanks', re.IGNORECASE)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce08787d-7124-49e0-b396-4e8c96d6ea22",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#counting occurrences of each match\n",
    "match_counts = data['Applications'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "306b1ffc-40a8-4ab4-8c45-2fe9099b4c57",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print ('\\nCounts of each applications')\n",
    "print(match_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f313622c-d703-4f5b-b179-2e6882f19092",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#most common match\n",
    "most_common_match = match_counts.idxmax()\n",
    "most_common_match_count = match_counts.max()\n",
    "print(f\"\\nMost common match: {most_common_match} with {most_common_match_count} occurrences\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e2cbafb-6ec7-4c9e-9234-17aacc1c7856",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#filter out empty matches\n",
    "filtered_data = data[data['Applications'] != '']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1df6a023-63c2-4e9d-b8b3-d67c0d0486c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count the occurrences of each match\n",
    "match_counts = filtered_data['Applications'].value_counts()\n",
    "\n",
    "# Display the counts of each match\n",
    "print(\"\\nCounts of each match:\")\n",
    "print(match_counts)\n",
    "\n",
    "# Find the most common match and its count\n",
    "most_common_match = match_counts.idxmax()\n",
    "most_common_match_count = match_counts.max()\n",
    "\n",
    "print(f\"\\nMost common match: {most_common_match} with {most_common_match_count} occurrences\")\n",
    "\n",
    "#select the top N most common matches\n",
    "N = 2\n",
    "top_n_matches = match_counts.head(N)\n",
    "\n",
    "print(f\"\\nTop {N} most common matches:\")\n",
    "print(top_n_matches)\n",
    "\n",
    "# Calculate the total number of occurrences for the top N matches\n",
    "total_top_n_matches = top_n_matches.sum()\n",
    "\n",
    "#print(f\"\\nTotal occurrences of the top {N} matches: {total_top_n_matches}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3edf91cf-8e56-419d-bb62-1b4039a0be60",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "543b1981-0537-4122-96cc-6d42a407ccc7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Word Cloud Visuals \n",
    "\n",
    "# Combining all the matches into a single string\n",
    "all_matches = ' '.join([' '.join(match) if isinstance(match, list) else match for match in filtered_data['Applications']])\n",
    "\n",
    "# Generating the word cloud\n",
    "wordcloud = WordCloud(width=800, height=400, background_color='white').generate(all_matches)\n",
    "\n",
    "# Displaying the word cloud\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.imshow(wordcloud, interpolation='bilinear')\n",
    "plt.axis('off')\n",
    "plt.title('Word Count of applications')\n",
    "plt.show()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdd7e8bf-7efd-4607-b86a-34fd192d2559",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Bar Plot of Most Common Applications\n",
    "\n",
    "# Flatten the list of matches and count the occurrences of each match\n",
    "flattened_matches = [match for sublist in filtered_data['Applications'] for match in (sublist if isinstance(sublist, list) else [sublist])]\n",
    "match_counts = Counter(flattened_matches)\n",
    "\n",
    "# Converting to a DataFrame for plotting\n",
    "match_counts_df = pd.DataFrame(match_counts.items(), columns=['Match', 'Frequency']).sort_values(by='Frequency', ascending=False)\n",
    "\n",
    "# Plot the top N most common words\n",
    "N = 6\n",
    "top_n_words = match_counts_df.head(N)\n",
    "\n",
    "plt.figure(figsize=(12, 7))\n",
    "sns.barplot(x='Frequency', y='Match', data=top_n_words, palette='viridis', hue='Match',legend=True, dodge=False)\n",
    "plt.xlabel('Frequency per ticket', fontsize=14)\n",
    "plt.ylabel('Applications', fontsize=15)\n",
    "plt.title(f'Top {N} Most Common Applications in User Submitted Tickets',fontsize=14)\n",
    "plt.xticks(fontsize=12, fontweight='bold')\n",
    "plt.yticks(fontsize=12, fontweight='bold')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b1dd6fc-5345-49b5-92bd-96fa79ce443c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#ISSUES\n",
    "\n",
    "# 2D PREP\n",
    "patterns2 = {\n",
    "    \"login\": r\"(?i)\\blogin\\b\",\n",
    "    \"license\": r\"(?i)\\blicense\\b\",\n",
    "    \"password\": r\"(?i)\\bpassword\\b\",\n",
    "    \"error\": r\"(?i)\\berror\\b\",\n",
    "    \"crash\": r\"(?i)\\bcrash\\b\",\n",
    "    \"configuration\": r\"(?i)\\bconfiguration\\b\",\n",
    "    \"backup\": r\"(?i)\\bbackup\\b\",\n",
    "    \"restore\": r\"(?i)\\brestore\\b\",\n",
    "    \"network outage\": r\"(?i)\\bnetwork\\boutage\\b\",\n",
    "    \"log-in\": r\"(?i)\\blog-in\\b\",\n",
    "    \"activation\": r\"activation\",\n",
    "    \"blocked\": r\"(?i)\\bblocked\\b\",\n",
    "    \"locked\": r\"(?i)\\blocked\\b\",\n",
    "    \"approval\": r\"(?i)\\bapproval\\b\",\n",
    "    \"mfa\": r\"(?i)\\bmfa\\b\",\n",
    "    \"quota\": r\"(?i)\\bquota\\b\",  \n",
    "    \"connect\": r\"(?i)\\bconnect\\b\",\n",
    "    \"compile\": r\"(?i)\\bcompile\\b\",\n",
    "    \"link\": r\"(?i)\\blink\\b\",\n",
    "    \"build\": r\"(?i)\\bbuild\\b\",\n",
    "    \"make\": r\"(?i)\\bmake\\b\",\n",
    "    \"timeout\": r\"(?i)\\btimeout\\b\",\n",
    "    \"freeze\": r\"(?i)\\bfreeze\\b\",\n",
    "    \"unresponsive\": r\"(?i)\\bunresponsive\\b\",\n",
    "    \"lag\": r\"(?i)\\blag\\b\",\n",
    "    \"broken\": r\"(?i)\\bbroken\\b\",\n",
    "    \"request\": r\"(?i)\\brequest\\b\",\n",
    "    \"module\": r\"(?i)\\bmodule\\b\",\n",
    "    \"slow\": r\"(?i)\\bslow\\b\",\n",
    "    \"profile\": r\"(?i)\\bprofile\\b\",\n",
    "    \"performance\": r\"(?i)\\bperformance\\b\",\n",
    "    \"libraries\": r\"(?i)\\blibraries\\b\",\n",
    "    \"mpi\": r\"(?i)\\bmpi\\b\",\n",
    "    \"job\": r\"(?i)\\bjob\\b\",\n",
    "    \"parameter\": r\"(?i)\\bparameter\\b\",\n",
    "    \"hang\": r\"(?i)\\bhang\\b\",\n",
    "    \"certificate\": r\"(?i)\\bcertificate\\b\",\n",
    "    \"automated\": r\"(?i)\\bautomated\\b\",\n",
    "    \"documentation\": r\"(?i)\\bdocumentation\\b\",\n",
    "    \"docs\": r\"(?i)\\bdocs\\b\",\n",
    "    \"stop\": r\"(?i)\\bstop\\b\",\n",
    "    \"wrong\": r\"(?i)\\bwrong\\b\",\n",
    "    \"session\": r\"(?i)\\bsession\\b\",\n",
    "    \"node\": r\"(?i)\\bnode\\b\",\n",
    "    \"hpe\": r\"(?i)\\bhpe\\b\",\n",
    "    \"wait\": r\"(?i)\\bwait\\b\",\n",
    "    \"policy\": r\"(?i)\\bpolicy\\b\",\n",
    "    \"queue\": r\"(?i)\\bqueue\\b\",\n",
    "    \"permissions\": r\"(?i)\\bpermissions\\b\",\n",
    "    \"mpich\": r\"(?i)\\bmpich\\b\",\n",
    "    \"makefile\": r\"(?i)\\bmakefile\\b\",\n",
    "    \"crontab\": r\"(?i)\\bcrontab\\b\",\n",
    "    \"scrontab\": r\"(?i)\\bscrontab\\b\",\n",
    "    \"disconnect\": r\"(?i)\\bdisconnect\\b\",\n",
    "    \"user\": r\"(?i)\\buser\\b\",\n",
    "    \"account\": r\"(?i)\\baccount\\b\",\n",
    "    \"collab\": r\"(?i)\\bcollab\\b\",\n",
    "    \"ssh\": r\"(?i)\\bssh\\b\",\n",
    "   # \"closed\": r\"(?i)\\bclosed\\b\",\n",
    "    \"save\": r\"(?i)\\bsave\\b\",\n",
    "    \"processes\": r\"(?i)\\bprocesses\\b\",\n",
    "    \"sharing\": r\"(?i)\\bsharing\\b\",\n",
    "    \"CFS\": r\"(?i)\\bCFS\\b\",\n",
    "    \"scratch\": r\"(?i)\\bscratch\\b\",\n",
    "    \"executable\": r\"(?i)\\bexecutable\\b\",\n",
    "    \"IO\": r\"(?i)\\bIO\\b\",\n",
    "    \"CPU\": r\"(?i)\\bCPU\\b\",\n",
    "    \"GPU\": r\"(?i)\\bGPU\\b\",\n",
    "    \"intel\": r\"(?i)\\bintel\\b\",\n",
    "    \"gnu\": r\"(?i)\\bgnu\\b\",\n",
    "    \"nvidia\": r\"(?i)\\bnvidia\\b\",\n",
    "    \"cray\": r\"(?i)\\bcray\\b\",\n",
    "    \"file\": r\"(?i)\\bfile\\b\",\n",
    "    \"qos\": r\"(?i)\\bqos\\b\",\n",
    "    \"OS\": r\"(?i)\\bOS\\b\",\n",
    "    \"shell\": r\"(?i)\\bshell\\b\",\n",
    "    \"bash\": r\"(?i)\\bbash\\b\",\n",
    "    \"csh\": r\"(?i)\\bcsh\\b\",\n",
    "    \"tsh\": r\"(?i)\\btsh\\b\",\n",
    "    \"ip\": r\"(?i)\\bip\\b\",\n",
    "    \"compute\": r\"(?i)\\bcompute\\b\",\n",
    "    \"workflow\": r\"(?i)\\bworkflow\\b\",\n",
    "    \"memory\": r\"(?i)\\bmemory\\b\",\n",
    "    \"utilization\": r\"(?i)\\butilization\\b\",\n",
    "    \"data\": r\"(?i)\\bdata\\b\",\n",
    "    \"application\": r\"(?i)\\bapplication\\b\",\n",
    "    \"framework\": r\"(?i)\\bframework\\b\",\n",
    "    \"realtime\": r\"(?i)\\brealtime\\b\",\n",
    "    \"allocation\": r\"(?i)\\ballocation\\b\",\n",
    "    \"reservation\": r\"(?i)\\breservation\\b\",\n",
    "    \"slurm\": r\"(?i)\\bslurm\\b\",\n",
    "    \"script\": r\"(?i)\\bscript\\b\",\n",
    "    \"group\": r\"(?i)\\bgroup\\b\",\n",
    "    \"membership\": r\"(?i)\\bmembership\\b\",\n",
    "    \"create\": r\"(?i)\\bcreate\\b\",\n",
    "    \"access\": r\"(?i)\\baccess\\b\",\n",
    "    \"issue\": r\"(?i)\\bissue\\b\",\n",
    "    \"source\": r\"(?i)\\bsource\\b\"\n",
    "\n",
    "    #\"alloacation_year\": r\"\\ballocation year\\b\"\n",
    "}\n",
    "\n",
    "compiled_patterns2 = {\n",
    "    name: re.compile(pattern, re.IGNORECASE)\n",
    "    for name, pattern in patterns2.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9b91de9-35a0-4655-be64-0433916b35d2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def match(string, patterns):\n",
    "    issues = []\n",
    "    for name, pattern in patterns.items():\n",
    "        #print(name)\n",
    "        #print(matches)\n",
    "        if pattern.search(string):\n",
    "            #print(\"found\", name)\n",
    "            issues.append(name) \n",
    "    return issues\n",
    "\n",
    "def f(x):\n",
    "    return match(x, compiled_patterns2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b2401dc-dc60-46a3-871a-d3ee8091422d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "names2 = process_map(f, data['text_data'], max_workers=32, chunksize=1000) #play with the numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad68ece2-0ffd-43bb-bc80-e0e0ec18634c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12f08c8d-c80c-4567-893d-8255258b2b80",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b574b212-97eb-46af-818d-aef57e38f5f8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.insert(2,\"Issues\",names2,True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddcf4b4e-3751-4e25-8890-8bc55cd3c842",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82eb21ff-fa5c-4014-b8c9-2628306fa7e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Word Cloud Visuals \n",
    "\n",
    "# Combining all the matches into a single string\n",
    "all_matches = ' '.join([' '.join(match) if isinstance(match, list) else match for match in data['Issues']])\n",
    "\n",
    "# Generating the word cloud\n",
    "wordcloud = WordCloud(width=800, height=400, background_color='white').generate(all_matches)\n",
    "\n",
    "# Displaying the word cloud\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.imshow(wordcloud, interpolation='bilinear')\n",
    "plt.axis('off')\n",
    "plt.title('Word Count of Issues')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fa74583-6b7f-47e7-828c-5c16ba6f843b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Bar Plot of Most Common Words\n",
    "\n",
    "# Flatten the list of matches and count the occurrences of each match\n",
    "flattened_matches = [match for sublist in data['Issues'] for match in (sublist if isinstance(sublist, list) else [sublist])]\n",
    "match_counts = Counter(flattened_matches)\n",
    "\n",
    "# Converting to a DataFrame for plotting\n",
    "match_counts_df = pd.DataFrame(match_counts.items(), columns=['Match', 'Frequency']).sort_values(by='Frequency', ascending=False)\n",
    "\n",
    "# Plot the top N most common words\n",
    "N = 6\n",
    "top_n_words = match_counts_df.head(N)\n",
    "\n",
    "plt.figure(figsize=(12, 7))\n",
    "sns.barplot(x='Frequency', y='Match', data=top_n_words, palette='viridis', hue='Match',legend=True, dodge=False)\n",
    "plt.xlabel('Frequency per ticket',fontsize=14)\n",
    "plt.ylabel('Issues', fontsize=14)\n",
    "plt.title(f'Top {N} Most Common Issues in User Submitted Tickets', fontsize=14)\n",
    "plt.xticks(fontsize=12, fontweight='bold')\n",
    "plt.yticks(fontsize=12, fontweight='bold')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "791254c6-4c68-4967-9cb5-0454b5ff7706",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.to_csv('/global/u2/m/masare/ticket-text-analysis/backup.csv', index=False)\n",
    "#data_df = pd.read_csv('/global/u2/m/masare/ticket-text-analysis/Ticket.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a9f365c-6a8a-4f13-8704-3bc3d0c29ee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#filter out empty matches\n",
    "filtered_data = data[data['Applications'] != '']\n",
    "\n",
    "applications_keywords = [match for sublist in filtered_data['Applications'] for match in (sublist if isinstance(sublist, list) else [sublist])]\n",
    "issues_keywords = flattened_matches = [match for sublist in data['Issues'] for match in (sublist if isinstance(sublist, list) else [sublist])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efd55d86-3d89-4cdf-887e-f27af8d95b25",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "issues_counts = data['Issues'].apply(Counter)\n",
    "#issues_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23bd6a99-2fff-4df9-9f51-63813394320c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "apps_counts = data['Applications'].apply(Counter)\n",
    "#apps_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca0c189e-3fa2-4c69-8fed-01295887ecd3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "issues_df =pd.DataFrame.from_records(issues_counts).fillna(0).astype(int)\n",
    "apps_df =pd.DataFrame.from_records(apps_counts).fillna(0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71e73433-2d8a-4c54-b08d-5934f806f066",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "issues_df['ID'] = data['ID']\n",
    "apps_df['ID'] = data['ID']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36925ca3-aa42-4708-bf70-bb8d6a2a3d05",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "issues_df.set_index('ID', inplace=True)\n",
    "#issues_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e65f488a-2e91-42e5-867b-3281bbff7984",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "apps_df.set_index('ID', inplace=True)\n",
    "#apps_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2491ef85-78c3-4faa-8023-e8be42aea8fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "###Functions. I had to continue here because the regex matched words  were not showing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ece0f5ec-5a1a-48eb-bf3c-d38b6bbd4689",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Flatten the issues and applications into a list of tuples (app, issue) and count occurrences\n",
    "import matplotlib.colors\n",
    "counts = []\n",
    "for idx, row in data.iterrows():\n",
    "    apps = row['Applications']\n",
    "    issues = row['Issues']\n",
    "    for app in apps:\n",
    "        for issue in issues:\n",
    "            counts.append((app, issue))\n",
    "\n",
    "\n",
    "#  counts to a DataFrame\n",
    "counts_df = pd.DataFrame(counts, columns=['Application', 'Issue'])\n",
    "counts_matrix = counts_df.groupby(['Application', 'Issue']).size().unstack(fill_value=0)\n",
    "\n",
    "\n",
    "def plot_heatmap(matrix):\n",
    "    \"\"\"\n",
    "    Plots a heatmap for the given matrix.\n",
    "    \n",
    "    Parameters:\n",
    "    matrix (pd.DataFrame): DataFrame representing the 2D matrix.\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(48, 32))\n",
    "    sns.heatmap(matrix, annot=False, fmt=\"d\", cmap=\"YlGnBu\", norm=matplotlib.colors.LogNorm())\n",
    "    plt.title(\"Occurrences of Issues in Applications\")\n",
    "    plt.xlabel(\"Issues\")\n",
    "    plt.ylabel(\"Applications\")\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# Plot the heatmap\n",
    "plot_heatmap(counts_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94abe472-20f9-4adb-a1cc-a73291e48ed5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#counts_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07708356-d67f-4505-8feb-94c961f533a4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#counts_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb7bcc8d-c603-4c8d-ba1e-b9803f7d3072",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Find the maximum and minimum entries in the matrix\n",
    "max_value = counts_matrix.max().max()\n",
    "min_value = counts_matrix.min().min()\n",
    "\n",
    "\n",
    "# Find the positions of the maximum and minimum values\n",
    "max_position = counts_matrix.stack().idxmax()\n",
    "min_position = counts_matrix.stack().idxmin()\n",
    "\n",
    "\n",
    "print(f\"Maximum value: {max_value} at position: {max_position}\")\n",
    "print(f\"Minimum value: {min_value} at position: {min_position}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0254c9e1-6c01-4673-af4e-6acafcc69bc2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Find the row with the maximum sum of entries\n",
    "row_sums = counts_matrix.sum(axis=1)\n",
    "max_row_sum = row_sums.max()\n",
    "max_row = row_sums.idxmax()\n",
    "\n",
    "\n",
    "print(f\"Row with the maximum sum: {max_row} with a sum of {max_row_sum}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecfb6877-0c5f-4ab5-92cd-a70aa0943b6e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Find the row with the minimum sum of entries\n",
    "min_row_sum = row_sums.min()\n",
    "min_row = row_sums.idxmin()\n",
    "\n",
    "\n",
    "print(f\"Row with the minimum sum: {min_row} with a sum of {min_row_sum}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efa4c57e-f3bf-4d5d-8297-859951e3c183",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "###Sample of 30\n",
    "\n",
    "data_sample = data.sample(n=4000, replace=True, random_state=1)\n",
    "\n",
    "\n",
    "# Flatten the issues and applications into a list of tuples (app, issue) and count occurrences\n",
    "counts = []\n",
    "for idx, row in data_sample.iterrows():\n",
    "    apps = row['Applications']\n",
    "    issues = row['Issues']\n",
    "    for app in apps:\n",
    "        for issue in issues:\n",
    "            counts.append((app, issue))\n",
    "\n",
    "\n",
    "# Convert the counts to a DataFrame\n",
    "counts_df = pd.DataFrame(counts, columns=['Application', 'Issue'])\n",
    "counts_matrix = counts_df.groupby(['Application', 'Issue']).size().unstack(fill_value=0)\n",
    "\n",
    "\n",
    "def plot_heatmap(matrix):\n",
    "    \"\"\"\n",
    "    Plots a heatmap for the given matrix.\n",
    "    \n",
    "    Parameters:\n",
    "    matrix (pd.DataFrame): DataFrame representing the 2D matrix.\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(12, 8))\n",
    "    sns.heatmap(matrix, annot=True, fmt=\"d\", cmap=\"coolwarm\")\n",
    "    plt.title(\"Occurrences of Issues in Applications for Each Submitted Ticket\")\n",
    "    plt.xlabel(\"Issues\")\n",
    "    plt.ylabel(\"Applications\")\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# Plot the heatmap\n",
    "plot_heatmap(counts_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90eaa61d-8bc5-45b0-9728-f7d63fa480f3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Find the row with the maximum sum of entries\n",
    "row_sums = counts_matrix.sum(axis=1)\n",
    "max_row_sum = row_sums.max()\n",
    "max_row = row_sums.idxmax()\n",
    "\n",
    "\n",
    "print(f\"Row with the maximum sum: {max_row} with a sum of {max_row_sum}\")\n",
    "\n",
    "\n",
    "# Find the row with the minimum sum of entries\n",
    "min_row_sum = row_sums.min()\n",
    "min_row = row_sums.idxmin()\n",
    "\n",
    "\n",
    "print(f\"Row with the minimum sum: {min_row} with a sum of {min_row_sum}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17516866-60fa-41ab-80dd-3e26a340a82e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Filter out zero entries\n",
    "counts_matrix_filtered = counts_matrix.loc[:, (counts_matrix != 0).any(axis=0)]\n",
    "counts_matrix_filtered = counts_matrix_filtered.loc[(counts_matrix_filtered != 0).any(axis=1), :]\n",
    "\n",
    "\n",
    "def plot_heatmap(matrix):\n",
    "    \"\"\"\n",
    "    Plots a heatmap for the given matrix.\n",
    "    \n",
    "    Parameters:\n",
    "    matrix (pd.DataFrame): DataFrame representing the 2D matrix.\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(12, 8))\n",
    "    sns.heatmap(matrix, annot=True, fmt=\"d\", cmap=\"viridis\")\n",
    "    plt.title(\"Occurrences of Issues in Applications\")\n",
    "    plt.xlabel(\"Issues\")\n",
    "    plt.ylabel(\"Applications\")\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# Plot the heatmap\n",
    "plot_heatmap(counts_matrix_filtered)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfc01e4b-d2b1-483f-b7f0-235af9df85f5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Find the row with the maximum sum of entries\n",
    "row_sums = counts_matrix_filtered.sum(axis=1)\n",
    "max_row_sum = row_sums.max()\n",
    "max_row = row_sums.idxmax()\n",
    "\n",
    "\n",
    "print(f\"Row with the maximum sum: {max_row} with a sum of {max_row_sum}\")\n",
    "\n",
    "\n",
    "# Find the column with the maximum sum of entries\n",
    "col_sums = counts_matrix_filtered.sum(axis=0)\n",
    "max_col_sum = col_sums.max()\n",
    "max_col = col_sums.idxmax()\n",
    "\n",
    "\n",
    "print(f\"Column with the maximum sum: {max_col} with a sum of {max_col_sum}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3342e813-3eac-49d1-8894-eac229560563",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "counts_matrix_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4122d9e-c588-4319-b2ce-87524fec8316",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "data_sample = data.sample(n=30, replace=True, random_state=1)\n",
    "# Filter out zero entries\n",
    "counts_matrix_filtered = counts_matrix.loc[:, (counts_matrix != 0).any(axis=0)]\n",
    "counts_matrix_filtered = counts_matrix_filtered.loc[(counts_matrix_filtered != 0).any(axis=1), :]\n",
    "\n",
    "\n",
    "def plot_heatmap(matrix):\n",
    "    \"\"\"\n",
    "    Plots a heatmap for the given matrix.\n",
    "    \n",
    "    Parameters:\n",
    "    matrix (pd.DataFrame): DataFrame representing the 2D matrix.\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(12, 8))\n",
    "    sns.heatmap(matrix, annot=True, fmt=\"d\", cmap=\"YlOrRd\")\n",
    "    plt.title(\"Occurrences of Issues in Applications\")\n",
    "    plt.xlabel(\"Issues\")\n",
    "    plt.ylabel(\"Applications\")\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# Plot the heatmap\n",
    "plot_heatmap(counts_matrix_filtered)\n",
    "\n",
    "\n",
    "# Find the row with the maximum sum of entries\n",
    "row_sums = counts_matrix_filtered.sum(axis=1)\n",
    "max_row_sum = row_sums.max()\n",
    "max_row = row_sums.idxmax()\n",
    "\n",
    "\n",
    "print(f\"Row with the maximum sum: {max_row} with a sum of {max_row_sum}\")\n",
    "\n",
    "\n",
    "# Find the column with the maximum sum of entries\n",
    "col_sums = counts_matrix_filtered.sum(axis=0)\n",
    "max_col_sum = col_sums.max()\n",
    "max_col = col_sums.idxmax()\n",
    "\n",
    "\n",
    "print(f\"Column with the maximum sum: {max_col} with a sum of {max_col_sum}\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ab1532d-19d6-402c-bf7e-9a8092e8a1e3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "counts_matrix_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a49e195b-63a9-4e14-86a0-953d814c321e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "data_sample = data.sample(n=40, replace=True, random_state=1)\n",
    "\n",
    "\n",
    "# Flatten the issues and applications into a list of tuples (app, issue) and count occurrences\n",
    "counts = []\n",
    "for idx, row in data_sample.iterrows():\n",
    "    apps = row['Applications']\n",
    "    issues = row['Issues']\n",
    "    for app in apps:\n",
    "        for issue in issues:\n",
    "            counts.append((app, issue))\n",
    "\n",
    "\n",
    "# Convert the counts to a DataFrame\n",
    "counts_df = pd.DataFrame(counts, columns=['Application', 'Issue'])\n",
    "counts_matrix = counts_df.groupby(['Application', 'Issue']).size().unstack(fill_value=0)\n",
    "\n",
    "\n",
    "# Filter out zero entries\n",
    "counts_matrix_filtered = counts_matrix.loc[:, (counts_matrix != 0).any(axis=0)]\n",
    "counts_matrix_filtered = counts_matrix_filtered.loc[(counts_matrix_filtered != 0).any(axis=1), :]\n",
    "\n",
    "\n",
    "def plot_heatmap(matrix):\n",
    "    \"\"\"\n",
    "    Plots a heatmap for the given matrix.\n",
    "    \n",
    "    Parameters:\n",
    "    matrix (pd.DataFrame): DataFrame representing the 2D matrix.\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(12, 8))\n",
    "    sns.heatmap(matrix, annot=True, fmt=\"d\", cmap=\"YlOrRd\", cbar_kws={'label': 'Count'})\n",
    "    plt.title(\"Occurrences of Issues in Applications\")\n",
    "    plt.xlabel(\"Issues\")\n",
    "    plt.ylabel(\"Applications\")\n",
    "    plt.xticks(rotation=45, ha='right')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# Plot the heatmap\n",
    "plot_heatmap(counts_matrix_filtered)\n",
    "\n",
    "\n",
    "# Find the row with the maximum sum of entries\n",
    "row_sums = counts_matrix_filtered.sum(axis=1)\n",
    "max_row_sum = row_sums.max()\n",
    "max_row = row_sums.idxmax()\n",
    "\n",
    "\n",
    "print(f\"Row with the maximum sum: {max_row} with a sum of {max_row_sum}\")\n",
    "\n",
    "\n",
    "# Find the column with the maximum sum of entries\n",
    "col_sums = counts_matrix_filtered.sum(axis=0)\n",
    "max_col_sum = col_sums.max()\n",
    "max_col = col_sums.idxmax()\n",
    "\n",
    "\n",
    "print(f\"Column with the maximum sum: {max_col} with a sum of {max_col_sum}\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93c294c9-7be8-4e40-8710-3b0b7cbcb135",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#poster\n",
    "dt = {\n",
    "    'crash': [43,26,34,1,8,1],\n",
    "    'error': [374,126,268,40,89,16],\n",
    "    'allocation': [64,16,91,42,21,11],\n",
    "    'license': [18,207,24,4,2,1],\n",
    "    'request': [162,199,157,56,42,14],\n",
    "    'login': [141,20,99,12,23,3],\n",
    "    'Application': ['Python','vasp', 'OpenMPI', 'E3SM', 'Shifter', 'WRF']\n",
    "}\n",
    "\n",
    "\n",
    "# Create DataFrame\n",
    "df = pd.DataFrame(dt)\n",
    "df.set_index('Application', inplace=True)\n",
    "\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# Create a heatmap\n",
    "plt.figure(figsize=(10, 6))  #  size of the figure\n",
    "sns.heatmap(df, annot=True, cmap='coolwarm', fmt='d',linewidths=0.5)\n",
    "plt.title('Matched Applications and issues for Each Ticket', fontweight='bold')\n",
    "plt.xlabel('Issues', fontweight='bold')\n",
    "plt.ylabel('Applications', fontweight='bold')\n",
    "plt.xticks(fontsize=12, fontweight='bold', rotation=45)\n",
    "plt.yticks(fontsize=12, fontweight='bold')\n",
    "plt.show()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e2b7dad-51c4-43ae-a747-903604718918",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "matrix = df.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "286df0dc-bf68-4f77-8fcd-8ea504790480",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b3d73c2-0825-4854-b67e-88e182c9bda2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sacct",
   "language": "python",
   "name": "sacct"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}