Merge branch 'dev' into UOT-130237

.gitignore

@@ -245,5 +245,6 @@ models/sent_index*
 models/qexp_model*
 models/transformers
 models/
+redis-data/
 # Ignore override for non local deployment
 docker-compose.override.yml

gamechangerml/api/docker-compose.yml

@@ -6,11 +6,13 @@ services:
     image: redis
     ports:
       - "6380:6380"
-    command: --port 6380
+    command: ["redis-server", "--appendonly", "yes","--port 6380"]
     expose:
       - "6380"
     networks:
       - app-net
+    volumes:
+      - ../redis-data:/data
   gamechanger-ml-gpu:
     container_name: gc-ml-gpu
     image: gamechanger-ml-gpu

gamechangerml/api/fastapi/routers/search.py

@@ -99,7 +99,8 @@ async def trans_sentence_infer(
         )
         logger.info(results)
     except Exception:
-        logger.error(f"Unable to get results from sentence transformer for {body}")
+        logger.error(
+            f"Unable to get results from sentence transformer for {body}")
         response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
         raise
     return results
@@ -155,7 +156,7 @@ async def post_expand_query_terms(body: dict, response: Response) -> dict:
     logger.info(f"Expanding: {body}")
     query_expander = (
         MODELS.query_expander
-        if body.get("qe_model", "gc_core") != "jbook"
+        if body.get("qe_model", "gc_core") != "jbook" or MODELS.query_expander_jbook==None
         else MODELS.query_expander_jbook
     )
     try:
@@ -204,28 +205,31 @@ async def post_word_sim(body: dict, response: Response) -> dict:
         logger.error(f"Error with query expansion on {terms}")
         response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
 
+
 @router.post("/recommender", status_code=200)
 async def post_recommender(body: dict, response: Response) -> dict:
     results = {}
     sample = False
     try:
-        filename = body["filename"]
-        if not filename:
+        filenames = body["filenames"]
+        if not filenames:
             if body["sample"]:
                 sample = body["sample"]
-        logger.info(f"Recommending similar documents to {filename}")
-        results = MODELS.recommender.get_recs(filename=filename, sample=sample)
+        logger.info(f"Recommending similar documents to {filenames}")
+        results = MODELS.recommender.get_recs(
+            filenames=filenames, sample=sample)
         if results['results'] != []:
             logger.info(f"Found similar docs: \n {str(results)}")
         else:
             logger.info("Did not find any similar docs")
     except Exception as e:
-        logger.warning(f"Could not get similar docs for {filename}")
+        logger.warning(f"Could not get similar docs for {filenames}")
         logger.warning(e)
         response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
-    
+
     return results
 
+
 def unquoted(term):
     """unquoted - unquotes string
     Args:

gamechangerml/api/tests/test_examples.py

@@ -1,6 +1,7 @@
 class TestSet:
     qa_test_data = {"text": "How manysides does a pentagon have?"}
-    qa_expect = {"answers":["five"],"question":"How many sides does a pentagon have?"}
+    qa_expect = {"answers": ["five"],
+                 "question": "How many sides does a pentagon have?"}
     text_extract_test_data = {
         "text": "In a major policy revision intended to encourage more schools to welcome children back to in-person instruction, federal health officials on Friday relaxed the six-foot distancing rule for elementary school students, saying they need only remain three feet apart in classrooms as long as everyone is wearing a mask. The three-foot rule also now applies to students in middle schools and high schools, as long as community transmission is not high, officials said. When transmission is high, however, these students must be at least six feet apart, unless they are taught in cohorts, or small groups that are kept separate from others. The six-foot rule still applies in the community at large, officials emphasized, and for teachers and other adults who work in schools, who must maintain that distance from other adults and from students. Most schools are already operating at least partially in person, and evidence suggests they are doing so relatively safely. Research shows in-school spread can be mitigated with simple safety measures such as masking, distancing, hand-washing and open windows. EDUCATION BRIEFING: The pandemic is upending education. Get the latest news and tips."
     }
@@ -74,16 +75,17 @@ class TestSet:
     ]
 
     word_sim_data = {"text": "naval command"}
-    word_sim_except ={
+    word_sim_except = {
         "naval": [
             "navy",
             "maritime"
         ],
-        'command':[]
+        'command': []
     }
 
-    recommender_data = {"filename": "Title 10"}
-    recommender_results = {'filename': 'Title 10', 'results':  ['Title 50', 'AACP 02.1', 'DoDD 5143.01 CH 2', 'DoDD S-5230.28', 'DoDI 5000.89']}
+    recommender_data = {"filenames": ["Title 10"]}
+    recommender_results = {'filenames': ['Title 10'], 'results':  [
+        'Title 50', 'AACP 02.1', 'DoDD 5143.01 CH 2', 'DoDD S-5230.28', 'DoDI 5000.89']}
 
     # extraction_data = {"text": "Carbon emissions trading is poised to go global, and billions of dollars — maybe even trillions — could be at stake. That's thanks to last month's U.N. climate summit in Glasgow Scotland, which approved a new international trading system where companies pay for cuts in greenhouse gas emissions somewhere else, rather than doing it themselves."}
     # extraction_keywords_expect = {
@@ -148,7 +150,7 @@ class TestSet:
             },
         ],
     }
-    transformer_list_expect =  {
+    transformer_list_expect = {
         'bert-base-cased-squad2',
         'distilbart-mnli-12-3',
         'distilbert-base-uncased-distilled-squad',
@@ -157,4 +159,4 @@ class TestSet:
         'msmarco-distilbert-base-v2_20220105'
         # 'msmarco-distilbert-base-v2_2021-10-17',
         # 'msmarco-distilbert-base-v2_20211210',
-        }
+    }

gamechangerml/api/utils/pathselect.py

@@ -44,7 +44,7 @@ def get_model_paths():
         else:
             print("defaulting INDEX_PATH to qexp")
             QEXP_JBOOK_MODEL_PATH = os.path.join(
-                Config.LOCAL_PACKAGED_MODELS_DIR, "jbook_qexp_20211029"
+                Config.LOCAL_PACKAGED_MODELS_DIR, "jbook_qexp_20220131"
             )
     except Exception as e:
         logger.error(e)

gamechangerml/api/utils/processmanager.py

@@ -34,13 +34,16 @@
 
     }
 
-    PROCESS_STATUS.value = {"flags": default_flags}
-    COMPLETED_PROCESS.value = []
 except Exception as e:
     print(e)
 
+if PROCESS_STATUS.value == None:
+    PROCESS_STATUS.value = {"flags": default_flags}
+if COMPLETED_PROCESS.value == None:
+    COMPLETED_PROCESS.value = []
+
 
-def update_status(key, progress=0, total=100, message="", failed=False,thread_id=""):
+def update_status(key, progress=0, total=100, message="", failed=False,thread_id="", completed_max = 20):
 
     try:
         if progress == total or failed:
@@ -63,6 +66,8 @@ def update_status(key, progress=0, total=100, message="", failed=False,thread_id
                         del running_threads[tempProcess['thread_id']]
                 if not failed:
                     completed_list = COMPLETED_PROCESS.value
+                    if len(completed_list) == completed_max :
+                        completed_list.pop(0)
                     completed_list.append(completed)
                     COMPLETED_PROCESS.value = completed_list
                 else: