new revision

README.md

@@ -1,2 +1,41 @@
 # Autocards
-Learn more by reading [the official write-up](https://psionica.org/docs/lab/autocards/).
+* Automatically create flashcards from user input, PDF files, wikipedia summary, webpages, and more!
+* To get a real world example, you can take a look at the complete output on [this article](https://www.biography.com/political-figure/philip-ii-of-macedon) can be found [in this folder](./output_example/). Nothing has been manually altered, it's the direct output.
+* Code is PEP compliant and all docstrings are written, hence contributions and PR are extremely appreciated
+* Learn more by reading [the official write-up](https://psionica.org/docs/lab/autocards/).
+
+## Install guide:
+* `git clone https//github.com/Psionica/Autocards`
+* `cd Autocards`
+* `pip install -r ./requirements.txt`
+* run a python console: `ipython3`
+* install punkt by running `!python -m nltk.downloader punkt`
+
+### Autocards usage
+```
+# loading
+from autocards import Autocards
+a = Autocards()
+
+# eating the input text using one of the following ways:
+a.consume_var(my_text, per_paragraph=True) 
+a.consume_user_input(title="")
+a.consume_wiki_summary(keyword, lang="en")
+a.consume_textfile(filename, per_paragraph=True)
+a.consume_pdf(pdf_path, per_paragraph=True)
+a.consume_web(source, mode="url", element="p")
+# =>  * element is the html element, like p for paragraph
+#     * mode can be "url" or "local"
+
+# three ways to get the results back:
+out = a.string_output(prefix='', jeopardy=False)
+#    =>  * prefix is a text that will be appended before the qa
+#        * jeopardy is when switching question and answer
+a.print(prefix='', jeopardy=False)
+a.pprint(prefix='', jeopardy=False)  # pretty printing
+df = a.pandas_output(prefix='')
+a.csv_export("output.csv", prefix="", jeopardy=False)
+a.json_export("output.json", prefix="", jeopardy=False)
+
+# Also note that a user provided his own terrible scripts that you can get inspiration from, they are located in the folder `examples_script`
+```

autocards.py

@@ -1,63 +1,276 @@
 from pipelines import qg_pipeline
-from transformers import pipeline
+
+from tqdm import tqdm
+from pathlib import Path
+import pandas as pd
+import time
+import signal
+import pdb
+import re
+import os
+from contextlib import suppress
+
+import requests
+import PyPDF2
+import wikipedia
+from wikipedia.exceptions import PageError
 from bs4 import BeautifulSoup
-import csv
+from pprint import pprint
+
+# otherwise csv and json outputs contain a warning string
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
 
 
 class Autocards:
     def __init__(self):
-        self.qg = qg_pipeline('question-generation', model='valhalla/t5-base-qg-hl', ans_model='valhalla/t5-small-qa-qg-hl')
+        print("Loading backend...")
+        self.qg = qg_pipeline('question-generation',
+                              model='valhalla/t5-base-qg-hl',
+                              ans_model='valhalla/t5-small-qa-qg-hl')
         self.qa_pairs = []
+        global n, cur_n
+        n = len(self.qa_pairs)
+        cur_n = n
+
+    def _call_qg(self, text, title):
+        """
+        Call question generation module, then turn the answer into a
+        dictionnary containing metadata (clozed formating, creation time,
+        title, source text)
+        """
+        try:
+            self.qa_pairs += self.qg(text)
+        except IndexError:
+            print(f"\nSkipping section because no cards \
+could be made from it:{text}\n")
+            self.qa_pairs.append({"question": "skipped",
+                                  "answer": "skipped"})
+
+        global n, cur_n
+        cur_n = len(self.qa_pairs)
+        diff = cur_n - n
+        n = len(self.qa_pairs)
+
+        cur_time = time.asctime()
+        for i in range(diff):
+            i += 1
+            cloze = self.qa_pairs[-i]['question']\
+                + "<br>{{c1::"\
+                + self.qa_pairs[-i]['answer']\
+                + "}}"
+            self.qa_pairs[-i] = {**self.qa_pairs[-i],
+                                 "clozed_text": cloze,
+                                 "creation_time": cur_time,
+                                 "title": title,
+                                 "source": text
+                                 }
+        tqdm.write(f"Added {diff} qa pair (total = {cur_n})")
+
+    def _sanitize_text(self, text):
+        "correct common errors in text"
+        # wikipedia style citation:
+        text = re.sub(r"\[\d*\]", "", text)
+        # extra spaces
+        text = re.sub("\s\s*", " ", text)
+        # new lines
+        text = re.sub("\n", ".", text)
+        text = re.sub("..*", ".", text)
+        return text
 
-    def consume_text(self, text, per_paragraph=False):
+    def consume_var(self, text, title="", per_paragraph=False):
+        "Take text as input and create qa pairs"
         text = text.replace('\xad ', '')
 
         if per_paragraph:
-            for paragraph in text.split('\n\n'):
-                self.qa_pairs += self.qg(paragraph)
+            for paragraph in tqdm(text.split('\n\n'),
+                                  desc="Processing by paragraph",
+                                  unit="paragraph"):
+                self._call_qg(paragraph, title)
         else:
-            text = text.replace('\n\n', '. ').replace('..', '.')
-            self.qa_pairs += self.qg(text)
+            text = self._sanitize_text(text)
+            self._call_qg(text, title)
+
+    def consume_user_input(self, title=""):
+        "Take user input and create qa pairs"
+        user_input = input("Enter your text below then press Enter (press\
+ enter twice to validate input):\n>")
+
+        print("\nFeeding your text to Autocards.")
+        user_input = self._sanitize_text(user_input)
+        self.consume_var(user_input, title, per_paragraph=False)
+        print("Done feeding text.")
+
+    def consume_wiki_summary(self, keyword, lang="en"):
+        "Take a wikipedia keyword and creates qa pairs from its summary"
+        if "http" in keyword:
+            print("To consume a wikipedia summmary, you have to input \
+the title of the article and not the url")
+            return None
+        wikipedia.set_lang(lang)
+        try:
+            wiki = wikipedia.page(keyword)
+        except PageError as e:
+            print(f"Page not found, error code:\n{e}")
+            return None
+        summary = wiki.summary
+        title = wiki.title
+        print(f"Article title: {title}")
+
+        summary = self._sanitize_text(summary)
+        self.consume_var(summary, title, True)
+
+    def consume_pdf(self, pdf_path, per_paragraph=True):
+        if not Path(pdf_path).exists():
+            print(f"PDF file not found at {pdf_path}!")
+            return None
+        pdf = PyPDF2.PdfFileReader(open(pdf_path, 'rb'))
+        try:
+            title = pdf.documentInfo['/Title']
+            print(f"PDF title : {title}")
+        except KeyError:
+            title = pdf_path.split("/")[-1]
+            print(f"PDF title : {title}")
+
+        full_text = []
+        for page in pdf.pages:
+            full_text.append(page.extractText())
+        text = " ".join(full_text)
+        text = self._sanitize_text(text)
 
-    def consume_text_file(self, filename):
-        self.consume_text(open(filename).read())
+        self.consume_var(text, title, per_paragraph)
 
-    def consume_paper(self, filename):
-        soup = BeautifulSoup(open(filename), 'xml')
-        paragraphs = []
+    def consume_textfile(self, filepath, per_paragraph=False):
+        "Take text file as input and create qa pairs"
+        if not Path(filepath).exists():
+            print(f"File not found at {filepath}")
+        text = open(filepath).read()
+        text = self._sanitize_text(text)
+        self.consume_var(text,
+                         filepath,
+                         per_paragraph=per_paragraph)
 
-        for paragraph in soup.article.body.find_all('p'):
-            paragraph = ' '.join(paragraph.get_text().split())
-            if len(paragraph) > 40:
-                paragraphs += [paragraph]
+    def consume_web(self, source, mode="url", element="p"):
+        "Take html file (local or via url) and create qa pairs"
+        if mode not in ["local", "url"]:
+            return "invalid arguments"
+        if mode == "local":
+            soup = BeautifulSoup(open(source), 'xml')
+        elif mode == "url":
+            res = requests.get(source, timeout=15)
+            html = res.content
+            soup = BeautifulSoup(html, 'xml')
 
-        qa_pairs = []
-        for paragraph in paragraphs:
-            qa_pairs += self.qg(paragraph)
+        try:
+            el = soup.article.body.find_all(element)
+        except AttributeError:
+            print("Using fallback method to extract page content")
+            el = soup.find_all(element)
 
-        self.qa_pairs += qa_pairs
+        title = ""
+        with suppress(Exception):
+            title = soup.find_all('h1')[0].text
+        with suppress(Exception):
+            title = soup.find_all('h1').text
+        with suppress(Exception):
+            title = soup.find_all('title').text
+        title.strip()
+        if title == "":
+            print(f"Couldn't find title of the page")
+            title = source
 
-    def clear(self):
+        valid_sections = []  # remove text sections that are too short:
+        for section in el:
+            section = ' '.join(section.get_text().split())
+            if len(section) > 40:
+                valid_sections += [section]
+            else:
+                print(f"Ignored string because too short: {section}")
+
+        if not valid_sections:
+            print("No valid sections found, change the 'element' argument\
+ to look for other html sections than 'p'")
+            return None
+
+        for section in tqdm(valid_sections,
+                            desc="Processing by section",
+                            unit="section"):
+            section = self._sanitize_text(section)
+            self._call_qg(section, title)
+
+    def clear_qa(self):
+        "Delete currently stored qa pairs"
         self.qa_pairs = []
+        global n, cur_n
+        n = 0
+        cur_n = n
 
-    def print(self, prefix='', jeopardy=False):
-        if prefix != '':
+    def string_output(self, prefix='', jeopardy=False):
+        "Return qa pairs to the user"
+        if prefix != "" and prefix[-1] != ' ':
             prefix += ' '
-
+        if len(self.qa_pairs) == 0:
+            print("No qa generated yet!")
+            return None
+
+        res = []
         for qa_pair in self.qa_pairs:
             if jeopardy:
-                print('\"' + prefix + qa_pair['answer'] + '\",\"' + qa_pair['question'] + '\"')
+                string = f"\"{prefix}{qa_pair['answer']}\",\"\
+{qa_pair['question']}\""
             else:
-                print('\"' + prefix + qa_pair['question'] + '\",\"' + qa_pair['answer'] + '\"')
+                string = f"\"{prefix}{qa_pair['question']}\",\"\
+{qa_pair['answer']}\""
+            res.append(string)
+        return res
+
+    def print(self, *args, **kwargs):
+        "Print qa pairs to the user"
+        print(self.string_output(*args, **kwargs))
+
+    def pprint(self, *args, **kwargs):
+        "Prettyprint qa pairs to the user"
+        pprint(self.string_output(*args, **kwargs))
 
-    def export(self, filename, prefix='', jeopardy=False):
-        if prefix != '':
+    def pandas_output(self, prefix=''):
+        if len(self.qa_pairs) == 0:
+            print("No qa generated yet!")
+            return None
+        "Output a Pandas DataFrame containing qa pairs and metadata"
+        df = pd.DataFrame(columns=list(self.qa_pairs[0].keys()))
+        for qa in self.qa_pairs:
+            df = df.append(qa, ignore_index=True)
+        for i in df.index:
+            for c in df.columns:
+                if pd.isna(df.loc[i, c]):
+                    # otherwise export functions break:
+                    df.loc[i, c] = "Error"
+        return df
+
+    def to_csv(self, filename, prefix='', jeopardy=False):
+        "Export qa pairs as csv file"
+        if len(self.qa_pairs) == 0:
+            print("No qa generated yet!")
+            return None
+        if prefix != "" and prefix[-1] != ' ':
             prefix += ' '
 
-        with open(filename, 'w', newline='') as file:
-            writer = csv.writer(file)
-            for qa_pair in self.qa_pairs:
-                if jeopardy:
-                    writer.writerow([prefix + qa_pair['answer'], qa_pair['question']])
-                else:
-                    writer.writerow([prefix + qa_pair['question'], qa_pair['answer']])
+        df = self.pandas_output(prefix)
+
+        for i in df.index:
+            for c in df.columns:
+                df.loc[i, c] = str(df.loc[i, c]).replace(",", r"\,")
+
+        df.to_csv(filename)
+        print(f"Done writing qa pairs to {filename}")
+
+    def to_json(self, filename, prefix='', jeopardy=False):
+        "Export qa pairs as json file"
+        if len(self.qa_pairs) == 0:
+            print("No qa generated yet!")
+            return None
+        if prefix != "" and prefix[-1] != ' ':
+            prefix += ' '
+
+        self.pandas_output(prefix).to_json(filename)
+        print(f"Done writing qa pairs to {filename}")
+

examples_script/README.md

@@ -0,0 +1 @@
+The folder `examples_script` is a collection of ad hoc scripts by users. They can probably not really used by anyone but provide some examples of how to use Autocards in the field for specific tasks