json-logging

Author: martingerlach

addlink-query_links.py

@@ -10,6 +10,16 @@
 
 PATH_mwaddlink=""
 
+## logging via json
+#https://github.com/bobbui/json-logging-python
+import json_logging, logging, sys
+LOG_LEVEL = logging.DEBUG
+# log is initialized without a web framework name
+json_logging.init_non_web(enable_json=True)
+logger = logging.getLogger("logger")
+logger.setLevel(LOG_LEVEL)
+logger.addHandler(logging.StreamHandler(sys.stdout))
+
 def main():
     parser = argparse.ArgumentParser()
 
@@ -30,65 +40,75 @@ def main():
                         type = float,
                         help="threshold value for links to be recommended")
 
-    parser.add_argument("--output","-o",
-                        default="",
-                        type=str,
-                        help="if None, print to terminal, otherwise write result to file")
-
     args = parser.parse_args()
     lang = args.lang.replace('wiki','')
     page_title = normalise_title(args.page)
     threshold = args.threshold
-    output_path = args.output
 
+    logger.info('Getting link recommendations for article %s in %swiki with link-threshold %s'%(page_title, lang,threshold))
+    
+    ## open the trained model
+    logger.info('Loading the trained model')
     try:
-
-        anchors = SqliteDict(os.path.join(PATH_mwaddlink,"data/{0}/{0}.anchors.sqlite".format(lang)) )
-        pageids = SqliteDict(os.path.join(PATH_mwaddlink,"data/{0}/{0}.pageids.sqlite".format(lang)) )
+        anchors = SqliteDict(os.path.join(PATH_mwaddlink,"data/{0}/{0}.anchors.sqlite".format(lang))  )
+        pageids = SqliteDict(os.path.join(PATH_mwaddlink,"data/{0}/{0}.pageids.sqlite".format(lang)))
         redirects = SqliteDict(os.path.join(PATH_mwaddlink,"data/{0}/{0}.redirects.sqlite".format(lang)) )
         word2vec = SqliteDict(os.path.join(PATH_mwaddlink,"data/{0}/{0}.w2v.filtered.sqlite".format(lang)) )
         nav2vec = SqliteDict(os.path.join(PATH_mwaddlink,"data/{0}/{0}.nav.filtered.sqlite".format(lang)) )
         ## load trained model
         n_cpus_max = min([int(multiprocessing.cpu_count()/4),8])
         model = xgb.XGBClassifier(n_jobs =n_cpus_max )  # init model
-        model.load_model(os.path.join(PATH_mwaddlink,"data/{0}/{0}.linkmodel.bin".format(lang)))  # load data
+        model.load_model(os.path.join(PATH_mwaddlink,"data/{0}/{0}.linkmodel_v2.bin".format(lang)))  # load data
     except:
-        print('Link recommendation model not available for %swiki. try another language.'%lang)
-
+        # logging
+        logger.error('Could not open trained model in %swiki. try another language.'%lang)
 
+    ## querying the API to get the wikitext for the page
+    logger.info('Getting the wikitext of the article')
     try:
         page_dict = getPageDict(page_title,lang)
         wikitext = page_dict['wikitext']
         pageid = page_dict['pageid']
         revid = page_dict['revid']
     except:
         wikitext = ""
-        print("""Not able to retrieve article '%s' in %swiki. try another article."""%(page_title,lang))
+        logger.error("""Not able to retrieve article '%s' in %swiki. try another article."""%(page_title,lang))
+
+    ## querying the API to get the wikitext for the page
+    logger.info('Processing wikitext to get link recommendations')
     try:
         added_links = process_page(wikitext, page_title, anchors, pageids, redirects, word2vec,nav2vec, model, threshold = threshold, return_wikitext = False)
     except:
-        print("""Not able to get links-recommendations  for article '%s' in %swiki. """%(page_title,lang))
-    anchors.close()
-    pageids.close()
-    redirects.close()
-    word2vec.close()
-    nav2vec.close()
+        logger.error("""Not able to process article '%s' in %swiki. try another article."""%(page_title,lang))
+
+    ## closing model
+    try:
+        anchors.close()
+        pageids.close()
+        redirects.close()
+        word2vec.close()
+        nav2vec.close()
+    except:
+        logger.warning('Could not close model in %swiki.'%lang)
+
 
+    ## querying the API to get the wikitext for the page
+    logger.info('Number of links from recommendation model: %s'%len(added_links))
+    if len(added_links) == 0:
+        logger.info('Model did not yield any links to recommend. Try a lower link-threshold (e.g. -t 0.2)')
+
     dict_return = {
         'page_title':page_title,
         'lang':lang,
         'pageid':pageid,
         'revid':revid,
         'no_added_links':len(added_links),
         'added_links':added_links,
-
     }
     json_out = json.dumps(dict_return, indent=4)
-    if len(output_path) == 0:
-        print(json_out)
-    else:
-        with open(output_path,'w') as fout:
-            fout.write(json_out+'\n')
+    logger.info('Recommended links: %s',dict_return)
+    print('--- Recommended links ---')
+    print(json_out)
 
 if __name__ == "__main__":
     main()

requirements.txt

@@ -23,13 +23,15 @@ jedi==0.17.2
 jieba==0.42.1
 Jinja2==2.11.2
 joblib==0.16.0
+json-logging==1.2.10
 jsonable==0.3.1
 jsonschema==3.2.0
 jupyter==1.0.0
 jupyter-client==6.1.7
 jupyter-console==6.2.0
 jupyter-core==4.6.3
 jupyterlab-pygments==0.1.2
+line-profiler==3.0.2
 lmdb==1.0.0
 marisa-trie==0.7.5
 MarkupSafe==1.1.1
@@ -86,6 +88,6 @@ wcwidth==0.2.5
 webencodings==0.5.1
 widgetsnbextension==3.5.1
 wikipedia2vec==1.0.4
-wikitextparser==0.45.1
+wikitextparser==0.44.0
 xgboost==1.2.0
 zipp==3.2.0

requirements_query.txt

@@ -3,6 +3,7 @@ chardet==3.0.4
 click==7.1.2
 idna==2.10
 joblib==0.16.0
+json-logging==1.2.10
 mwparserfromhell==0.5.4
 nltk==3.5
 numpy==1.19.2